Blame view

tests/test_query_parser_mixed_language.py 3.48 KB
a8261ece   tangwang   检索效果优化
1
2
3
4
5
6
7
8
9
  from config import FunctionScoreConfig, IndexConfig, QueryConfig, RerankConfig, SPUConfig, SearchConfig
  from query.query_parser import QueryParser
  
  
  class _DummyTranslator:
      def translate(self, text, target_lang, source_lang, scene, model_name):
          return f"{text}-{target_lang}"
  
  
ef5baa86   tangwang   混杂语言处理
10
11
12
13
  def _tokenizer(text):
      return str(text).split()
  
  
6823fe3e   tangwang   feat(search): 混合语...
14
15
16
17
18
19
20
21
  def test_pure_english_word_token_length_and_script():
      assert QueryParser._is_pure_english_word_token("ab") is False
      assert QueryParser._is_pure_english_word_token("abc") is True
      assert QueryParser._is_pure_english_word_token("wi-fi") is True
      assert QueryParser._is_pure_english_word_token("连衣裙") is False
      assert QueryParser._is_pure_english_word_token("ab12") is False
  
  
a8261ece   tangwang   检索效果优化
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
  def _build_config() -> SearchConfig:
      return SearchConfig(
          es_index_name="test_products",
          field_boosts={"title.en": 3.0, "title.zh": 3.0},
          indexes=[IndexConfig(name="default", label="default", fields=["title.en", "title.zh"])],
          query_config=QueryConfig(
              enable_text_embedding=False,
              enable_query_rewrite=False,
              supported_languages=["en", "zh"],
              default_language="zh",
          ),
          function_score=FunctionScoreConfig(),
          rerank=RerankConfig(),
          spu_config=SPUConfig(enabled=False),
      )
  
  
  def test_parse_adds_en_fields_for_mixed_chinese_query_with_meaningful_english(monkeypatch):
ef5baa86   tangwang   混杂语言处理
40
      parser = QueryParser(_build_config(), translator=_DummyTranslator(), tokenizer=_tokenizer)
a8261ece   tangwang   检索效果优化
41
      monkeypatch.setattr(parser.language_detector, "detect", lambda text: "zh")
a8261ece   tangwang   检索效果优化
42
  
ef5baa86   tangwang   混杂语言处理
43
44
45
46
47
48
      result = parser.parse(
          "法式 dress 连衣裙",
          tenant_id="162",
          generate_vector=False,
          target_languages=["zh", "en"],
      )
a8261ece   tangwang   检索效果优化
49
50
  
      assert result.detected_language == "zh"
6823fe3e   tangwang   feat(search): 混合语...
51
52
      assert result.contains_chinese is True
      assert result.contains_english is True
ef5baa86   tangwang   混杂语言处理
53
54
55
56
      assert result.translations == {"en": "法式 dress 连衣裙-en"}
      assert result.query_tokens == ["法式", "dress", "连衣裙"]
      assert not hasattr(result, "query_text_by_lang")
      assert not hasattr(result, "search_langs")
a8261ece   tangwang   检索效果优化
57
58
59
  
  
  def test_parse_adds_zh_fields_for_english_query_when_cjk_present(monkeypatch):
ef5baa86   tangwang   混杂语言处理
60
      parser = QueryParser(_build_config(), translator=_DummyTranslator(), tokenizer=_tokenizer)
a8261ece   tangwang   检索效果优化
61
      monkeypatch.setattr(parser.language_detector, "detect", lambda text: "en")
a8261ece   tangwang   检索效果优化
62
  
ef5baa86   tangwang   混杂语言处理
63
64
65
66
67
68
      result = parser.parse(
          "red 连衣裙",
          tenant_id="0",
          generate_vector=False,
          target_languages=["en", "zh"],
      )
a8261ece   tangwang   检索效果优化
69
70
  
      assert result.detected_language == "en"
6823fe3e   tangwang   feat(search): 混合语...
71
72
      assert result.contains_chinese is True
      assert result.contains_english is True
ef5baa86   tangwang   混杂语言处理
73
74
      assert result.translations == {"zh": "red 连衣裙-zh"}
      assert result.query_tokens == ["red", "连衣裙"]
1556989b   tangwang   query翻译等待超时逻辑
75
76
77
78
  
  
  def test_parse_waits_for_translation_when_source_in_index_languages(monkeypatch):
      """en 在 index_languages 内时仍应等待并采纳 en->zh 翻译结果(与向量共用预算)。"""
ef5baa86   tangwang   混杂语言处理
79
      parser = QueryParser(_build_config(), translator=_DummyTranslator(), tokenizer=_tokenizer)
1556989b   tangwang   query翻译等待超时逻辑
80
      monkeypatch.setattr(parser.language_detector, "detect", lambda text: "en")
1556989b   tangwang   query翻译等待超时逻辑
81
  
ef5baa86   tangwang   混杂语言处理
82
83
84
85
86
87
      result = parser.parse(
          "off shoulder top",
          tenant_id="0",
          generate_vector=False,
          target_languages=["en", "zh"],
      )
1556989b   tangwang   query翻译等待超时逻辑
88
89
  
      assert result.detected_language == "en"
6823fe3e   tangwang   feat(search): 混合语...
90
91
      assert result.contains_chinese is False
      assert result.contains_english is True
1556989b   tangwang   query翻译等待超时逻辑
92
      assert result.translations.get("zh") == "off shoulder top-zh"
ef5baa86   tangwang   混杂语言处理
93
      assert not hasattr(result, "source_in_index_languages")