tangwang · tangwang · tangwang
Showing 11 changed files Show diff stats
config/loader.py
config/schema.py
docs/TODO.txt
docs/常用查询 - ES.md
embeddings/server.py
frontend/static/css/style.css
frontend/static/js/app.js
query/language_detector.py
requirements.txt
scripts/start_embedding_service.sh
search/es_query_builder.py
@@ -281,8 +281,8 @@ class AppConfigLoader:
                     ["title", "brief", "vendor", "category_name_text"],
                 )
             ),
-            base_minimum_should_match=str(text_strategy.get("base_minimum_should_match", "75%")),
-            translation_minimum_should_match=str(text_strategy.get("translation_minimum_should_match", "75%")),
+            base_minimum_should_match=str(text_strategy.get("base_minimum_should_match", "70%")),
+            translation_minimum_should_match=str(text_strategy.get("translation_minimum_should_match", "70%")),
             translation_boost=float(text_strategy.get("translation_boost", 0.4)),
             translation_boost_when_source_missing=float(
                 text_strategy.get("translation_boost_when_source_missing", 1.0)
@@ -51,8 +51,8 @@ class QueryConfig:
     core_multilingual_fields: List[str] = field(
         default_factory=lambda: ["title", "brief", "vendor", "category_name_text"]
     )
-    base_minimum_should_match: str = "75%"
-    translation_minimum_should_match: str = "75%"
+    base_minimum_should_match: str = "70%"
+    translation_minimum_should_match: str = "70%"
     translation_boost: float = 0.4
     translation_boost_when_source_missing: float = 1.0
     source_boost_when_missing: float = 0.6
@@ -236,14 +236,19 @@ config/environments/&lt;env&gt;.yaml
+筛选SKU： 先只筛选第一个维度，但考虑到用户搜索词可能带了尺码，所以第二、三个维度也要考虑
+引入图片的相关性：
+图片的向量最好做SKU维度，用 SPU 维度还是 SKU 维度？
+1. SKU维度（主款式，option1维度），如果用户搜索“蓝色 T恤”，这种图片相关性会比较有价值。
+2. 我不考虑颜色的差异，其余的款式一般是大小之类的。这些图片，项链细粉到 SKU 维度，可能价值不大，性价比偏低
-
-
+无结果重查
+稀有语言，翻译可能超时（因为zh-en互译之外的翻译耗时更长）
@@ -654,3 +654,18 @@ GET /search_products_tenant_170/_search
         }
     }
 }
+
+
+检查某个字段是否存在
+curl -u 'saas:4hOaLaf41y2VuI8y' -X POST \
+  'http://localhost:9200/search_products_tenant_163/_count' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "query": {
+      "bool": {
+        "filter": [
+          { "exists": { "field": "title_embedding" } }
+        ]
+      }
+    }
+  }'
 \ No newline at end of file
@@ -14,7 +14,6 @@ import time
 import uuid
 from collections import deque
 from dataclasses import dataclass
-from logging.handlers import TimedRotatingFileHandler
 from typing import Any, Dict, List, Optional
 import numpy as np
@@ -44,9 +43,7 @@ def configure_embedding_logging() -&gt; None:
         return
     log_dir = pathlib.Path("logs")
-    verbose_dir = log_dir / "verbose"
     log_dir.mkdir(exist_ok=True)
-    verbose_dir.mkdir(parents=True, exist_ok=True)
     log_level = os.getenv("LOG_LEVEL", "INFO").upper()
     numeric_level = getattr(logging, log_level, logging.INFO)
@@ -56,47 +53,18 @@ def configure_embedding_logging() -&gt; None:
     request_filter = _DefaultRequestIdFilter()
     root_logger.setLevel(numeric_level)
-
-    file_handler = TimedRotatingFileHandler(
-        filename=log_dir / "embedding_api.log",
-        when="midnight",
-        interval=1,
-        backupCount=30,
-        encoding="utf-8",
-    )
-    file_handler.setLevel(numeric_level)
-    file_handler.setFormatter(formatter)
-    file_handler.addFilter(request_filter)
-    root_logger.addHandler(file_handler)
-
-    error_handler = TimedRotatingFileHandler(
-        filename=log_dir / "embedding_api_error.log",
-        when="midnight",
-        interval=1,
-        backupCount=30,
-        encoding="utf-8",
-    )
-    error_handler.setLevel(logging.ERROR)
-    error_handler.setFormatter(formatter)
-    error_handler.addFilter(request_filter)
-    root_logger.addHandler(error_handler)
+    root_logger.handlers.clear()
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(numeric_level)
+    stream_handler.setFormatter(formatter)
+    stream_handler.addFilter(request_filter)
+    root_logger.addHandler(stream_handler)
     verbose_logger = logging.getLogger("embedding.verbose")
     verbose_logger.setLevel(numeric_level)
     verbose_logger.handlers.clear()
-    verbose_logger.propagate = False
-
-    verbose_handler = TimedRotatingFileHandler(
-        filename=verbose_dir / "embedding_verbose.log",
-        when="midnight",
-        interval=1,
-        backupCount=30,
-        encoding="utf-8",
-    )
-    verbose_handler.setLevel(numeric_level)
-    verbose_handler.setFormatter(formatter)
-    verbose_handler.addFilter(request_filter)
-    verbose_logger.addHandler(verbose_handler)
+    # Consolidate verbose logs into the main embedding log stream.
+    verbose_logger.propagate = True
     root_logger._embedding_logging_configured = True  # type: ignore[attr-defined]
@@ -379,7 +379,7 @@ body {
     margin-top: 8px;
 }
-.product-debug-inline-es-btn {
+.product-debug-inline-result-btn {
     font-family: inherit;
     font-size: 12px;
     padding: 4px 10px;
@@ -390,27 +390,22 @@ body {
     cursor: pointer;
 }
-.product-debug-inline-es-btn:hover {
+.product-debug-inline-result-btn:hover {
     background: #f0f0f0;
     border-color: #bbb;
 }
-.product-debug--es-expanded {
+.product-debug--result-expanded {
     max-height: min(70vh, 720px);
 }
-.product-es-doc-panel {
+.product-result-doc-panel {
     margin-top: 10px;
     padding-top: 8px;
     border-top: 1px dashed #e8e8e8;
 }
-.product-es-doc-panel-status {
-    font-size: 12px;
-    color: #888;
-}
-
-.product-es-doc-pre {
+.product-result-doc-pre {
     margin: 6px 0 0;
     padding: 10px;
     background: #f5f5f5;
@@ -68,25 +68,25 @@ function initializeApp() {
     // 初始化租户下拉框和分面面板
     console.log('Initializing app...');
     initTenantSelect();
-    setupProductGridEsDocToggle();
+    setupProductGridResultDocToggle();
     const searchInput = document.getElementById('searchInput');
     if (searchInput) {
         searchInput.focus();
     }
 }
-/** Delegated handler: toggle inline ES raw response under each result card (survives innerHTML refresh on re-search). */
-function setupProductGridEsDocToggle() {
+/** Delegated handler: toggle inline current result JSON under each result card (survives innerHTML refresh on re-search). */
+function setupProductGridResultDocToggle() {
     const grid = document.getElementById('productGrid');
-    if (!grid || grid.dataset.esDocToggleBound === '1') {
+    if (!grid || grid.dataset.resultDocToggleBound === '1') {
         return;
     }
-    grid.dataset.esDocToggleBound = '1';
-    grid.addEventListener('click', onProductGridEsDocToggleClick);
+    grid.dataset.resultDocToggleBound = '1';
+    grid.addEventListener('click', onProductGridResultDocToggleClick);
 }
-async function onProductGridEsDocToggleClick(event) {
-    const btn = event.target.closest('[data-action="toggle-es-inline-doc"]');
+function onProductGridResultDocToggleClick(event) {
+    const btn = event.target.closest('[data-action="toggle-result-inline-doc"]');
     if (!btn) {
         return;
     }
@@ -95,55 +95,27 @@ async function onProductGridEsDocToggleClick(event) {
     if (!debugRoot) {
         return;
     }
-    const panel = debugRoot.querySelector('.product-es-doc-panel');
-    const pre = debugRoot.querySelector('.product-es-doc-pre');
-    const statusEl = debugRoot.querySelector('.product-es-doc-panel-status');
-    if (!panel || !pre || !statusEl) {
+    const panel = debugRoot.querySelector('.product-result-doc-panel');
+    const pre = debugRoot.querySelector('.product-result-doc-pre');
+    if (!panel || !pre) {
         return;
     }
-    const spuId = btn.getAttribute('data-spu-id') || '';
-    const tenantId = getTenantId();
-    const url = `${API_BASE_URL}/search/es-doc/${encodeURIComponent(spuId)}?tenant_id=${encodeURIComponent(tenantId)}`;
-
-    if (debugRoot.dataset.esInlineOpen === '1') {
+    if (debugRoot.dataset.resultInlineOpen === '1') {
         panel.setAttribute('hidden', '');
-        debugRoot.classList.remove('product-debug--es-expanded');
-        debugRoot.dataset.esInlineOpen = '0';
-        btn.textContent = '在结果中显示 ES 文档';
+        debugRoot.classList.remove('product-debug--result-expanded');
+        debugRoot.dataset.resultInlineOpen = '0';
+        btn.textContent = '在结果中显示当前结果数据';
         return;
     }
     panel.removeAttribute('hidden');
-    debugRoot.classList.add('product-debug--es-expanded');
-    debugRoot.dataset.esInlineOpen = '1';
-    btn.textContent = '隐藏 ES 文档';
-
-    if (pre.textContent.length > 0) {
-        panel.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
-        return;
-    }
-
-    statusEl.style.display = '';
-    statusEl.textContent = '加载中…';
-    pre.style.display = 'none';
-
-    try {
-        const response = await fetch(url);
-        if (!response.ok) {
-            const errText = await response.text();
-            throw new Error(`HTTP ${response.status}: ${errText.slice(0, 200)}`);
-        }
-        const data = await response.json();
-        pre.textContent = customStringify(data);
-        statusEl.style.display = 'none';
-        pre.style.display = 'block';
-    } catch (err) {
-        console.error('ES doc fetch failed', err);
-        statusEl.textContent = `加载失败: ${err.message || err}`;
-        pre.style.display = 'none';
+    debugRoot.classList.add('product-debug--result-expanded');
+    debugRoot.dataset.resultInlineOpen = '1';
+    btn.textContent = '隐藏当前结果数据';
+    if (pre.textContent.length === 0) {
+        pre.textContent = btn.getAttribute('data-result-json') || '{}';
     }
-
     panel.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
 }
@@ -213,7 +185,7 @@ function initTenantSelect() {
         });
         // 设置默认值（仅当输入框为空时）
         if (!tenantSelect.value.trim()) {
-            tenantSelect.value = availableTenants.includes('170') ? '170' : availableTenants[0];
+            tenantSelect.value = availableTenants.includes('0') ? '0' : availableTenants[0];
         }
     }
@@ -462,6 +434,7 @@ function displayResults(data) {
                 });
             }
+            const resultJson = customStringify(result);
             const rawUrl = `${API_BASE_URL}/search/es-doc/${encodeURIComponent(spuId)}?tenant_id=${encodeURIComponent(tenantId)}`;
             debugHtml = `
@@ -475,18 +448,17 @@ function displayResults(data) {
                     <div class="product-debug-line">Fused score: ${fusedScore}</div>
                     ${titleLines}
                     <div class="product-debug-actions">
-                        <button type="button" class="product-debug-inline-es-btn"
-                            data-action="toggle-es-inline-doc"
-                            data-spu-id="${escapeAttr(String(spuId || ''))}">
-                            在结果中显示 ES 文档
+                        <button type="button" class="product-debug-inline-result-btn"
+                            data-action="toggle-result-inline-doc"
+                            data-result-json="${escapeAttr(resultJson)}">
+                            在结果中显示当前结果数据
                         </button>
                         <a class="product-debug-link" href="${rawUrl}" target="_blank" rel="noopener noreferrer">
                             查看 ES 原始文档
                         </a>
                     </div>
-                    <div class="product-es-doc-panel" hidden>
-                        <div class="product-es-doc-panel-status"></div>
-                        <pre class="product-es-doc-pre"></pre>
+                    <div class="product-result-doc-panel" hidden>
+                        <pre class="product-result-doc-pre"></pre>
                     </div>
                 </div>
             `;
 """
 Language detection utility.
-Detects language of short e-commerce queries with script checks + lightweight
-Latin-language scoring (de/fr/es/it/pt/nl/en).
+Script-first rules for CJK and other non-Latin scripts, then Lingua
+(lingua-language-detector) for Latin text and Romance/Germanic disambiguation.
 """
-from typing import Dict, List
+from __future__ import annotations
+
+from typing import Dict, Optional
 import re
+from lingua import Language, LanguageDetectorBuilder
+
+_LINGUA_TO_CODE: Dict[Language, str] = {
+    Language.CHINESE: "zh",
+    Language.ENGLISH: "en",
+    Language.JAPANESE: "ja",
+    Language.KOREAN: "ko",
+    Language.GERMAN: "de",
+    Language.FRENCH: "fr",
+    Language.SPANISH: "es",
+    Language.ITALIAN: "it",
+    Language.PORTUGUESE: "pt",
+    Language.DUTCH: "nl",
+    Language.RUSSIAN: "ru",
+    Language.ARABIC: "ar",
+    Language.HINDI: "hi",
+    Language.HEBREW: "he",
+    Language.THAI: "th",
+}
+
+_LINGUA_LANGUAGES = tuple(_LINGUA_TO_CODE.keys())
+
+_lingua_detector: Optional[object] = None
+
+
+def _get_lingua_detector():
+    global _lingua_detector
+    if _lingua_detector is None:
+        _lingua_detector = LanguageDetectorBuilder.from_languages(
+            *_LINGUA_LANGUAGES
+        ).build()
+    return _lingua_detector
+
 class LanguageDetector:
-    """Rule-based language detector for common e-commerce query languages."""
+    """Language detector: script hints + Lingua for Latin-family queries."""
     def __init__(self):
         self._re_zh = re.compile(r"[\u4e00-\u9fff]")
@@ -21,47 +56,6 @@ class LanguageDetector:
         self._re_hi = re.compile(r"[\u0900-\u097f]")
         self._re_he = re.compile(r"[\u0590-\u05ff]")
         self._re_th = re.compile(r"[\u0e00-\u0e7f]")
-        self._re_latin_word = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ]+")
-
-        # Stopwords + e-commerce terms for Latin-family disambiguation.
-        self._latin_lexicons: Dict[str, set] = {
-            "en": {
-                "the", "and", "for", "with", "new", "women", "men", "kids",
-                "shoe", "shoes", "dress", "shirt", "jacket", "bag", "wireless",
-            },
-            "de": {
-                "der", "die", "das", "und", "mit", "für", "damen", "herren",
-                "kinder", "schuhe", "kleid", "hemd", "jacke", "tasche",
-            },
-            "fr": {
-                "le", "la", "les", "et", "avec", "pour", "femme", "homme",
-                "enfant", "chaussures", "robe", "chemise", "veste", "sac",
-            },
-            "es": {
-                "el", "la", "los", "las", "y", "con", "para", "mujer", "hombre",
-                "niño", "niña", "zapatos", "vestido", "camisa", "chaqueta", "bolso",
-            },
-            "it": {
-                "il", "lo", "la", "gli", "le", "e", "con", "per", "donna", "uomo",
-                "bambino", "scarpe", "abito", "camicia", "giacca", "borsa",
-            },
-            "pt": {
-                "o", "a", "os", "as", "e", "com", "para", "mulher", "homem",
-                "criança", "sapatos", "vestido", "camisa", "jaqueta", "bolsa",
-            },
-            "nl": {
-                "de", "het", "en", "met", "voor", "dames", "heren", "kinderen",
-                "schoenen", "jurk", "overhemd", "jas", "tas",
-            },
-        }
-        self._diacritic_weights: Dict[str, Dict[str, int]] = {
-            "de": {"ä": 3, "ö": 3, "ü": 3, "ß": 4},
-            "fr": {"é": 2, "è": 2, "ê": 2, "à": 2, "ç": 2, "ù": 2, "ô": 2},
-            "es": {"ñ": 3, "á": 2, "é": 2, "í": 2, "ó": 2, "ú": 2},
-            "it": {"à": 2, "è": 2, "é": 2, "ì": 2, "ò": 2, "ù": 2},
-            "pt": {"ã": 3, "õ": 3, "ç": 2, "á": 2, "â": 2, "ê": 2, "ô": 2},
-            "nl": {"ij": 2},
-        }
     def detect(self, text: str) -> str:
         """
@@ -71,9 +65,9 @@ class LanguageDetector:
         """
         if not text or not text.strip():
             return "unknown"
-        q = text.strip().lower()
+        q = text.strip()
-        # Script-first detection for non-Latin languages.
+        # Script-first: unambiguous blocks before Latin/Romance Lingua pass.
         if self._re_ja_kana.search(q):
             return "ja"
         if self._re_ko.search(q):
@@ -91,48 +85,11 @@ class LanguageDetector:
         if self._re_th.search(q):
             return "th"
-        # Latin-family scoring.
-        tokens = self._re_latin_word.findall(q)
-        if not tokens:
+        detected = _get_lingua_detector().detect_language_of(q)
+        if detected is None:
             return "unknown"
-
-        scores: Dict[str, float] = {k: 0.0 for k in self._latin_lexicons.keys()}
-        scores["en"] = scores.get("en", 0.0)
-        token_set = set(tokens)
-
-        # Lexicon matches
-        for lang, lex in self._latin_lexicons.items():
-            overlap = len(token_set & lex)
-            if overlap:
-                scores[lang] += overlap * 2.0
-
-        # Diacritics / orthographic hints
-        for lang, hints in self._diacritic_weights.items():
-            for marker, weight in hints.items():
-                if marker in q:
-                    scores[lang] += weight
-
-        # Light suffix hints for common product words
-        for t in tokens:
-            if t.endswith("ung") or t.endswith("chen"):
-                scores["de"] += 0.6
-            if t.endswith("ción") or t.endswith("ado") or t.endswith("ada"):
-                scores["es"] += 0.6
-            if t.endswith("zione") or t.endswith("etto") or t.endswith("ella"):
-                scores["it"] += 0.6
-            if t.endswith("ção") or t.endswith("mente"):
-                scores["pt"] += 0.6
-            if t.endswith("ment") or t.endswith("eau"):
-                scores["fr"] += 0.5
-
-        # Fallback preference: English for pure Latin short tokens.
-        scores["en"] += 0.2
-
-        best_lang = max(scores.items(), key=lambda x: x[1])[0]
-        best_score = scores[best_lang]
-        if best_score <= 0:
-            return "en"
-        return best_lang
+        code = _LINGUA_TO_CODE.get(detected)
+        return code if code is not None else "unknown"
     def is_chinese(self, text: str) -> bool:
         return self.detect(text) == "zh"
@@ -42,3 +42,6 @@ click&gt;=8.1.0
 pytest>=7.4.0
 pytest-asyncio>=0.21.0
 httpx>=0.24.0
+
+# language detector
+lingua-language-detector
 \ No newline at end of file
@@ -138,7 +138,11 @@ fi
 if [[ "${IMAGE_MODEL_ENABLED}" == "1" ]]; then
   echo "Image max inflight: ${IMAGE_MAX_INFLIGHT:-1}"
 fi
-echo "Logs: logs/embedding_api.log, logs/embedding_api_error.log, logs/verbose/embedding_verbose.log"
+if [[ "${SERVICE_KIND}" == "image" ]]; then
+  echo "Logs: logs/embedding-image.log"
+else
+  echo "Logs: logs/embedding.log"
+fi
 echo
 echo "Tips:"
 echo "  - Use a single worker (GPU models cannot be safely duplicated across workers)."
@@ -153,12 +157,16 @@ echo
 UVICORN_LOG_LEVEL="${EMBEDDING_UVICORN_LOG_LEVEL:-info}"
 UVICORN_ACCESS_LOG="${EMBEDDING_UVICORN_ACCESS_LOG:-true}"
+UVICORN_LOG_CONFIG="${EMBEDDING_UVICORN_LOG_CONFIG:-${PROJECT_ROOT}/config/uvicorn_embedding_logging.json}"
 UVICORN_ARGS=(
   --host "${EMBEDDING_SERVICE_HOST}"
   --port "${EMBEDDING_SERVICE_PORT}"
   --workers 1
   --log-level "${UVICORN_LOG_LEVEL}"
 )
+if [[ -f "${UVICORN_LOG_CONFIG}" ]]; then
+  UVICORN_ARGS+=(--log-config "${UVICORN_LOG_CONFIG}")
+fi
 if [[ "${UVICORN_ACCESS_LOG}" == "0" || "${UVICORN_ACCESS_LOG}" == "false" || "${UVICORN_ACCESS_LOG}" == "no" ]]; then
   UVICORN_ARGS+=(--no-access-log)
 fi
@@ -33,8 +33,8 @@ class ESQueryBuilder:
         function_score_config: Optional[FunctionScoreConfig] = None,
         default_language: str = "en",
         knn_boost: float = 0.25,
-        base_minimum_should_match: str = "75%",
-        translation_minimum_should_match: str = "75%",
+        base_minimum_should_match: str = "70%",
+        translation_minimum_should_match: str = "70%",
         translation_boost: float = 0.4,
         translation_boost_when_source_missing: float = 1.0,
         source_boost_when_missing: float = 0.6,
@@ -261,16 +261,13 @@ class ESQueryBuilder:
             if parsed_query:
                 query_tokens = getattr(parsed_query, 'query_tokens', None) or []
                 token_count = len(query_tokens)
-                if token_count <= 2:
-                    knn_k, knn_num_candidates = 30, 100
-                    knn_boost = self.knn_boost * 0.6  # Lower weight for short queries
-                elif token_count >= 5:
-                    knn_k, knn_num_candidates = 80, 300
+                if token_count >= 5:
+                    knn_k, knn_num_candidates = 160, 500
                     knn_boost = self.knn_boost * 1.4  # Higher weight for long queries
                 else:
-                    knn_k, knn_num_candidates = 50, 200
+                    knn_k, knn_num_candidates = 120, 400
             else:
-                knn_k, knn_num_candidates = 50, 200
+                knn_k, knn_num_candidates = 120, 400
             knn_clause = {
                 "field": self.text_embedding_field,
                 "query_vector": query_vector.tolist(),