from query.tokenization import QueryTextAnalysisCache
  

  import pytest
  
  pytestmark = [pytest.mark.query]
  

  
  def test_han_coarse_tokens_follow_model_tokens_instead_of_whole_sentence():
      cache = QueryTextAnalysisCache(
          tokenizer=lambda text: [("路上", 0, 2), ("穿着", 2, 4), ("女性", 4, 6), ("黑色", 10, 12)]
      )
      cache.set_language_hint("路上穿着女性的衣服是黑色的", "zh")
  
      tokenized = cache.get_tokenized_text("路上穿着女性的衣服是黑色的")
  
      assert tokenized.fine_tokens == ("路上", "穿着", "女性", "黑色")
      assert tokenized.coarse_tokens == ("路上", "穿着", "女性", "黑色")