From 99b72698b556ae19a00ba4cb6206a1343f033abe Mon Sep 17 00:00:00 2001 From: tangwang Date: Mon, 20 Apr 2026 12:55:04 +0800 Subject: [PATCH] 测试回归钩子梳理 --- CLAUDE.md | 43 +++++++++++++++++++++++++++---------------- docs/DEVELOPER_GUIDE.md | 15 ++++++++++----- docs/测试Pipeline说明.md | 542 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pytest.ini | 30 ++++++++++++++++++++++++++++++ scripts/run_ci_tests.sh | 15 +++++++++++++-- scripts/run_regression_tests.sh | 26 ++++++++++++++++++++++++++ search/searcher.py | 5 +++++ search/sku_intent_selector.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------ tests/ci/test_service_api_contracts.py | 2 ++ tests/conftest.py | 259 +++++++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ tests/manual/test_cnclip_service.py | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/manual/test_facet_api.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_cache_keys.py | 4 ++++ tests/test_cnclip_service.py | 143 ----------------------------------------------------------------------------------------------------------------------------------------------- tests/test_embedding_pipeline.py | 8 ++++++-- tests/test_embedding_service_limits.py | 2 ++ tests/test_embedding_service_priority.py | 6 ++++++ tests/test_es_query_builder.py | 4 ++++ tests/test_es_query_builder_text_recall_languages.py | 21 +++++++++++++++------ tests/test_eval_framework_clients.py | 2 ++ tests/test_eval_metrics.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------- tests/test_facet_api.py | 131 ----------------------------------------------------------------------------------------------------------------------------------- tests/test_keywords_query.py | 115 ------------------------------------------------------------------------------------------------------------------- tests/test_llm_enrichment_batch_fill.py | 4 ++++ tests/test_process_products_batching.py | 4 ++++ tests/test_product_enrich_partial_mode.py | 33 +++++++++++++++++++++++++++------ tests/test_product_title_exclusion.py | 4 ++++ tests/test_query_parser_mixed_language.py | 4 ++++ tests/test_rerank_client.py | 4 ++++ tests/test_rerank_provider_topn.py | 4 ++++ tests/test_rerank_query_text.py | 4 ++++ tests/test_reranker_dashscope_backend.py | 2 ++ tests/test_reranker_qwen3_gguf_backend.py | 4 ++++ tests/test_reranker_server_topn.py | 4 ++++ tests/test_search_evaluation_datasets.py | 4 ++++ tests/test_search_rerank_window.py | 4 ++++ tests/test_sku_intent_selector.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_style_intent.py | 4 ++++ tests/test_suggestions.py | 15 ++------------- tests/test_tokenization.py | 4 ++++ tests/test_translation_converter_resolution.py | 2 ++ tests/test_translation_deepl_backend.py | 4 ++++ tests/test_translation_llm_backend.py | 4 ++++ tests/test_translation_local_backends.py | 2 ++ tests/test_translator_failure_semantics.py | 2 ++ translation/prompts.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ translation/scenes.py | 4 ++++ 47 files changed, 867 insertions(+), 1204 deletions(-) create mode 100644 pytest.ini create mode 100755 scripts/run_regression_tests.sh create mode 100755 tests/manual/test_cnclip_service.py create mode 100755 tests/manual/test_facet_api.py delete mode 100755 tests/test_cnclip_service.py delete mode 100755 tests/test_facet_api.py delete mode 100644 tests/test_keywords_query.py diff --git a/CLAUDE.md b/CLAUDE.md index 97c3257..b03348d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,18 +99,29 @@ python main.py serve --host 0.0.0.0 --port 6002 --reload ### Testing ```bash -# Run all tests -pytest tests/ +# CI gate (API contracts + search core regression anchors) +./scripts/run_ci_tests.sh + +# Full regression anchor suite (pre-release / pre-merge) +./scripts/run_regression_tests.sh + +# Subsystem-scoped regression (e.g. search / query / intent / rerank / embedding / translation / indexer / suggestion) +SUBSYSTEM=rerank ./scripts/run_regression_tests.sh -# Run focused regression sets -python -m pytest tests/ci -q +# Whole automated suite +python -m pytest tests/ -q + +# Focused debugging pytest tests/test_rerank_client.py pytest tests/test_query_parser_mixed_language.py -# Test search from command line +# Command-line smoke python main.py search "query" --tenant-id 1 --size 10 ``` +See `docs/测试Pipeline说明.md` for the authoritative test pipeline guide, +including the regression hook matrix and marker conventions. + ### Development Utilities ```bash # Stop all services @@ -218,24 +229,24 @@ The system uses centralized configuration through `config/config.yaml`: ## Testing Infrastructure -**Test Framework**: pytest with async support +**Framework**: pytest. Authoritative guide: `docs/测试Pipeline说明.md`. + +**Layout**: +- `tests/` — flat file layout; each file targets one subsystem. +- `tests/ci/` — API / service contract tests (FastAPI `TestClient` with fake backends). +- `tests/manual/` — scripts that need live services (pytest does **not** collect these). +- `tests/conftest.py` — sys.path injection only. No global fixtures; all fakes live next to the tests that use them. -**Test Structure**: -- `tests/conftest.py`: Comprehensive test fixtures and configuration -- `tests/unit/`: Unit tests for individual components -- `tests/integration/`: Integration tests for system workflows -- Test markers: `@pytest.mark.unit`, `@pytest.mark.integration`, `@pytest.mark.api` +**Markers** (registered in `pytest.ini`, enforced by `--strict-markers`): +- Subsystem: `contract`, `search`, `query`, `intent`, `rerank`, `embedding`, `translation`, `indexer`, `suggestion`, `eval`. +- Regression gate: `regression` — anchor tests mandatory for `run_regression_tests.sh`. **Test Data**: - Tenant1: Mock data with 10,000 product records - Tenant2: CSV-based test dataset - Automated test data generation via `scripts/mock_data.sh` -**Key Test Fixtures** (from `conftest.py`): -- `sample_search_config`: Complete configuration for testing -- `mock_es_client`: Mocked Elasticsearch client -- `test_searcher`: Searcher instance with mock dependencies -- `temp_config_file`: Temporary YAML configuration for tests +**Principle**: tests must inject fakes for ES / DeepL / LLM / Redis. Never add tests that rely on real external services to the automated suite — put them under `tests/manual/`. ## API Endpoints diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index 17b8ee9..ad4c572 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -386,11 +386,16 @@ services: ### 8.2 测试 -- **位置**:`tests/`,可按 `unit/`、`integration/` 或按模块划分子目录;公共 fixture 在 `conftest.py`。 -- **标记**:使用 `@pytest.mark.unit`、`@pytest.mark.integration`、`@pytest.mark.api` 等区分用例类型,便于按需运行。 -- **依赖**:单元测试通过 mock(如 `mock_es_client`、`sample_search_config`)不依赖真实 ES/DB;集成测试需在说明中注明依赖服务。 -- **运行**:`python -m pytest tests/`;推荐最小回归:`python -m pytest tests/ci -q`;按模块聚焦可直接指定具体测试文件。 -- **原则**:新增逻辑应有对应测试;修改协议或配置契约时更新相关测试与 fixture。 +测试流水线的权威说明见 [`docs/测试Pipeline说明.md`](./测试Pipeline说明.md)。核心约定: + +- **位置**:`tests/` 下按文件平铺,`tests/ci/` 放 API 契约测试,`tests/manual/` 放需人工起服务的联调脚本(pytest 默认不 collect)。 +- **Marker**:`pytest.ini` 里登记了子系统 marker(`search / query / intent / rerank / embedding / translation / indexer / suggestion / eval / contract`)与 `regression` marker;新测试必须贴对应 marker(`--strict-markers` 会强制)。 +- **依赖**:测试一律通过注入 fake stub 隔离 ES / DeepL / LLM / Redis 等外部依赖。需要真实依赖的脚本放 `tests/manual/`。 +- **运行**: + - CI 门禁:`./scripts/run_ci_tests.sh`(契约 + search 回归锚点) + - 发版前:`./scripts/run_regression_tests.sh`(全部 `regression` 锚点;可配 `SUBSYSTEM=`) + - 全量:`python -m pytest tests/ -q` +- **原则**:新增逻辑应有对应测试;修改协议或配置契约时**同步**更新契约测试。不要在测试里保留"旧 assert 作为兼容"——请直接面向当前实现写断言,失败即意味着契约已变更,需要上层决策。 ### 8.3 配置与环境 diff --git a/docs/测试Pipeline说明.md b/docs/测试Pipeline说明.md index 8cd054f..f70db2e 100644 --- a/docs/测试Pipeline说明.md +++ b/docs/测试Pipeline说明.md @@ -1,508 +1,98 @@ # 搜索引擎测试流水线指南 -## 概述 +本文档是测试套件的**权威入口**,涵盖目录约定、运行方式、回归锚点矩阵、以及手动 +联调脚本的分工。任何与这里不一致的历史文档(例如提到 `tests/unit/` 或 +`scripts/start_test_environment.sh`)都是过期信息,以本文为准。 -本文档介绍了搜索引擎项目的完整测试流水线,包括测试环境搭建、测试执行、结果分析等内容。测试流水线设计用于commit前的自动化质量保证。 - -## 🏗️ 测试架构 - -### 测试层次 +## 1. 测试目录与分层 ``` -测试流水线 -├── 代码质量检查 (Code Quality) -│ ├── 代码格式化检查 (Black, isort) -│ ├── 静态分析 (Flake8, MyPy, Pylint) -│ └── 安全扫描 (Safety, Bandit) -│ -├── 单元测试 (Unit Tests) -│ ├── RequestContext测试 -│ ├── Searcher测试 -│ ├── QueryParser测试 -│ └── BooleanParser测试 -│ -├── 集成测试 (Integration Tests) -│ ├── 端到端搜索流程测试 -│ ├── 多组件协同测试 -│ └── 错误处理测试 -│ -├── API测试 (API Tests) -│ ├── REST API接口测试 -│ ├── 参数验证测试 -│ ├── 并发请求测试 -│ └── 错误响应测试 -│ -└── 性能测试 (Performance Tests) - ├── 响应时间测试 - ├── 并发性能测试 - └── 资源使用测试 +tests/ +├── conftest.py # 只做 sys.path 注入;不再维护全局 fixture +├── ci/ # API/服务契约(FastAPI TestClient + 全 fake 依赖) +│ └── test_service_api_contracts.py +├── manual/ # 需真实服务才能跑的联调脚本,pytest 默认不 collect +│ ├── test_build_docs_api.py +│ ├── test_cnclip_service.py +│ └── test_facet_api.py +└── test_*.py # 子系统单测(全部自带 fake,无外部依赖) ``` -### 核心组件 - -1. **RequestContext**: 请求级别的上下文管理器,用于跟踪测试过程中的所有数据 -2. **测试环境管理**: 自动化启动/停止测试依赖服务 -3. **测试执行引擎**: 统一的测试运行和结果收集 -4. **报告生成系统**: 多格式的测试报告生成 - -## 🚀 快速开始 +关键约束(写在 `pytest.ini` 里,不要另起分支): -### 本地测试环境 +- `testpaths = tests`,`norecursedirs = tests/manual`; +- `--strict-markers`:所有 marker 必须先在 `pytest.ini::markers` 登记; +- 测试**不得**依赖真实 ES / DeepL / LLM 服务。需要外部依赖的脚本请放 `tests/manual/`。 -1. **启动测试环境** - ```bash - # 启动所有必要的测试服务 - ./scripts/start_test_environment.sh - ``` +## 2. 运行方式 -2. **运行完整测试套件** - ```bash - # 运行所有测试 - python scripts/run_tests.py +| 场景 | 命令 | 覆盖范围 | +|------|------|----------| +| CI 门禁(每次提交) | `./scripts/run_ci_tests.sh` | `tests/ci` + `contract` marker + `search ∧ regression` | +| 发版 / 大合并前 | `./scripts/run_regression_tests.sh` | 所有 `@pytest.mark.regression` | +| 子系统子集 | `SUBSYSTEM=search ./scripts/run_regression_tests.sh` | 指定子系统的 regression 锚点 | +| 全量(含非回归) | `python -m pytest tests/ -q` | 全部自动化用例 | +| 手动联调 | `python tests/manual/