From 1852e3e352053c40124a172ed363ff42e40427e2 Mon Sep 17 00:00:00 2001 From: tangwang Date: Thu, 13 Nov 2025 12:09:36 +0800 Subject: [PATCH] 添加Base配置演示流程和数据库配置 --- .env | 24 ++++++++++++++---------- config/env_config.py | 21 +++++++++++++++++++++ frontend/base.html | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/create_base_frontend.py | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/demo_base.sh | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/frontend_server.py | 13 ++++++++++++- scripts/stop_base.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 597 insertions(+), 11 deletions(-) create mode 100644 frontend/base.html create mode 100755 scripts/create_base_frontend.py create mode 100755 scripts/demo_base.sh create mode 100755 scripts/stop_base.sh diff --git a/.env b/.env index 76c730b..49d9691 100644 --- a/.env +++ b/.env @@ -1,18 +1,15 @@ -# Environment Configuration - ACTIVE -# This file contains the actual configuration values - -# Elasticsearch Configuration (v8.18) +# Elasticsearch Configuration ES_HOST=http://localhost:9200 -ES_USERNAME=essa -ES_PASSWORD=4hOaLaf41y2VuI8y +ES_USERNAME= +ES_PASSWORD= -# Redis Configuration (for caching) +# Redis Configuration (Optional) REDIS_HOST=localhost REDIS_PORT=6479 -REDIS_PASSWORD=BMfv5aI31kgHWtlx +REDIS_PASSWORD= # DeepL Translation API -DEEPL_AUTH_KEY=c9293ab4-ad25-479b-919f-ab4e63b429ed +DEEPL_AUTH_KEY= # Customer Configuration CUSTOMER_ID=customer1 @@ -21,7 +18,14 @@ CUSTOMER_ID=customer1 API_HOST=0.0.0.0 API_PORT=6002 -# Embedding Models +# MySQL Database Configuration (Shoplazza) +DB_HOST=120.79.247.228 +DB_PORT=3316 +DB_DATABASE=saas +DB_USERNAME=saas +DB_PASSWORD=P89cZHS5d7dFyc9R + +# Model Directories TEXT_MODEL_DIR=/data/tw/models/bge-m3 IMAGE_MODEL_DIR=/data/tw/models/cn-clip diff --git a/config/env_config.py b/config/env_config.py index 316a3e9..ec22898 100644 --- a/config/env_config.py +++ b/config/env_config.py @@ -45,6 +45,15 @@ IMAGE_MODEL_DIR = os.getenv('IMAGE_MODEL_DIR', '/data/tw/models/cn-clip') # Cache Directory CACHE_DIR = os.getenv('CACHE_DIR', '.cache') +# MySQL Database Configuration (Shoplazza) +DB_CONFIG = { + 'host': os.getenv('DB_HOST', '120.79.247.228'), + 'port': int(os.getenv('DB_PORT', 3316)), + 'database': os.getenv('DB_DATABASE', 'saas'), + 'username': os.getenv('DB_USERNAME', 'saas'), + 'password': os.getenv('DB_PASSWORD', 'P89cZHS5d7dFyc9R'), +} + def get_es_config() -> Dict[str, Any]: """Get Elasticsearch configuration.""" @@ -66,6 +75,11 @@ def get_customer_id() -> str: return CUSTOMER_ID +def get_db_config() -> Dict[str, Any]: + """Get MySQL database configuration.""" + return DB_CONFIG.copy() + + def print_config(): """Print current configuration (with sensitive data masked).""" print("=" * 60) @@ -99,6 +113,13 @@ def print_config(): print("\nCache:") print(f" Cache Directory: {CACHE_DIR}") + print("\nMySQL Database:") + print(f" Host: {DB_CONFIG['host']}") + print(f" Port: {DB_CONFIG['port']}") + print(f" Database: {DB_CONFIG['database']}") + print(f" Username: {DB_CONFIG['username']}") + print(f" Password: {'*' * 10 if DB_CONFIG['password'] else 'None'}") + print("=" * 60) diff --git a/frontend/base.html b/frontend/base.html new file mode 100644 index 0000000..596163a --- /dev/null +++ b/frontend/base.html @@ -0,0 +1,89 @@ + + + + + + 店匠通用搜索 - Base Configuration + + + +
+ +
+
+ + 0 products found +
+
+ +
+
+ + +
+ + +
+ + +
+ +
+
Categories:
+
+
+ + +
+
Vendor:
+
+
+ + +
+
Tags:
+
+
+ + +
+
Price Range:
+
+
+ + +
+ +
+
+ + +
+
+
+

Welcome to Shoplazza Base Search

+

Enter keywords to search for products

+
+
+
+ + + + + + +
+ + + + + diff --git a/scripts/create_base_frontend.py b/scripts/create_base_frontend.py new file mode 100755 index 0000000..e77a0aa --- /dev/null +++ b/scripts/create_base_frontend.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Create frontend JavaScript file for base configuration. +""" + +import sys +import os +import argparse +import re +from pathlib import Path + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +def create_base_frontend_js(tenant_id: str, api_port: int = 6002, output_file: str = "frontend/static/js/app_base.js"): + """ + Create frontend JavaScript file for base configuration. + + Args: + tenant_id: Tenant ID + api_port: API port + output_file: Output file path + """ + # Read original app.js + original_file = Path(__file__).parent.parent / "frontend/static/js/app.js" + if not original_file.exists(): + print(f"ERROR: Original frontend file not found: {original_file}") + return 1 + + with open(original_file, 'r', encoding='utf-8') as f: + content = f.read() + + # Replace API_BASE_URL + api_url = f"http://localhost:{api_port}" + content = content.replace( + "const API_BASE_URL = 'http://120.76.41.98:6002';", + f"const API_BASE_URL = '{api_url}';" + ) + + # Add tenant_id constant at the beginning + content = content.replace( + "const API_BASE_URL =", + f"const TENANT_ID = '{tenant_id}';\nconst API_BASE_URL =" + ) + + # Update facets for base configuration + base_facets = ''' const facets = [ + { + "field": "category_keyword", + "size": 15, + "type": "terms" + }, + { + "field": "vendor_keyword", + "size": 15, + "type": "terms" + }, + { + "field": "tags_keyword", + "size": 10, + "type": "terms" + }, + { + "field": "min_price", + "type": "range", + "ranges": [ + {"key": "0-50", "to": 50}, + {"key": "50-100", "from": 50, "to": 100}, + {"key": "100-200", "from": 100, "to": 200}, + {"key": "200+", "from": 200} + ] + } + ];''' + + # Find and replace facets definition (multiline match) + facets_pattern = r'const facets = \[.*?\];' + content = re.sub(facets_pattern, base_facets, content, flags=re.DOTALL) + + # Update fetch to include tenant_id header + content = content.replace( + "headers: {\n 'Content-Type': 'application/json',\n },", + f"headers: {{\n 'Content-Type': 'application/json',\n 'X-Tenant-ID': TENANT_ID,\n }}," + ) + + # Replace hits with results throughout + content = re.sub(r'\bdata\.hits\b', 'data.results', content) + content = re.sub(r'!data\.hits', '!data.results', content) + + # Replace hit loop with product loop + content = re.sub( + r'data\.hits\.forEach\(\(hit\) => \{', + 'data.results.forEach((product) => {', + content + ) + + # Remove source extraction lines + content = re.sub(r'const source = hit\._source;\s*\n', '', content) + content = re.sub(r'const score = hit\._custom_score \|\| hit\._score;\s*\n', 'const score = product.relevance_score;\n', content) + + # Replace all source. references with product. + content = re.sub(r'\bsource\.', 'product.', content) + + # Replace specific field names for base configuration + # imageUrl -> image_url + content = re.sub(r'product\.imageUrl', 'product.image_url', content) + # name -> title + content = re.sub(r'product\.name', 'product.title', content) + content = re.sub(r'product\.enSpuName', 'product.title', content) + # categoryName -> category + content = re.sub(r'product\.categoryName', 'product.category', content) + # brandName -> vendor + content = re.sub(r'product\.brandName', 'product.vendor', content) + # price -> price (already correct) + # Remove moq and quantity fields (not in base config) + content = re.sub(r'
.*?
\s*\n', '', content, flags=re.DOTALL) + content = re.sub(r'
.*?
\s*\n', '', content, flags=re.DOTALL) + + # Add stock and variants display + # Find the product-price div and add stock info after it + stock_info = '''
+ ${product.in_stock ? 'In Stock' : 'Out of Stock'} + ${product.variants && product.variants.length > 0 ? `(${product.variants.length} variants)` : ''} +
+ +''' + content = re.sub( + r'(
.*?
\s*\n)', + r'\1' + stock_info, + content, + flags=re.DOTALL + ) + + # Update price display format + content = re.sub( + r'\$\{product\.price \? `\$\{product\.price\} ₽` : \'N/A\'\}', + '${product.price ? `$${product.price.toFixed(2)}` : \'N/A\'}', + content + ) + + # Add compare_at_price if exists + content = re.sub( + r'(\$\{product\.price \? `\$\$\{product\.price\.toFixed\(2\)\}` : \'N/A\'\})', + r'\1${product.compare_at_price && product.compare_at_price > product.price ? `$${product.compare_at_price.toFixed(2)}` : \'\'}', + content + ) + + # Update product-meta to use base config fields + content = re.sub( + r'
\s*\$\{product\.category \? escapeHtml\(product\.category\) : \'\'\}\s*\$\{product\.vendor \? \' \| \' \+ escapeHtml\(product\.vendor\) : \'\'\}\s*
', + '
${product.vendor ? escapeHtml(product.vendor) : \'\'}${product.product_type ? \' | \' + escapeHtml(product.product_type) : \'\'}${product.category ? \' | \' + escapeHtml(product.category) : \'\'}
', + content + ) + + # Remove create_time display (not in base config) + content = re.sub( + r'\$\{product\.create_time \? `.*?\s*` : \'\'\}', + '', + content, + flags=re.DOTALL + ) + + # Add tags display if exists + tags_display = ''' ${product.tags ? ` +
+ Tags: ${escapeHtml(product.tags)} +
+ ` : ''}''' + + # Add tags before closing product-card div + content = re.sub( + r'(\s*\s*`;\s*\n\s*\}\);)', + tags_display + r'\n \n `;\n });', + content, + count=1 + ) + + # Update displayFacets for base configuration field names + content = re.sub( + r"facet\.field === 'categoryName_keyword'", + "facet.field === 'category_keyword'", + content + ) + content = re.sub( + r"facet\.field === 'brandName_keyword'", + "facet.field === 'vendor_keyword'", + content + ) + content = re.sub( + r"facet\.field === 'supplierName_keyword'", + "facet.field === 'tags_keyword'", + content + ) + + # Write output file + output_path = Path(__file__).parent.parent / output_file + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, 'w', encoding='utf-8') as f: + f.write(content) + + print(f"Created base frontend JavaScript: {output_path}") + return 0 + + +def main(): + parser = argparse.ArgumentParser(description='Create frontend JavaScript for base configuration') + parser.add_argument('--tenant-id', default='1', help='Tenant ID') + parser.add_argument('--api-port', type=int, default=6002, help='API port') + parser.add_argument('--output', default='frontend/static/js/app_base.js', help='Output file') + + args = parser.parse_args() + + return create_base_frontend_js(args.tenant_id, args.api_port, args.output) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/demo_base.sh b/scripts/demo_base.sh new file mode 100755 index 0000000..2ee1f84 --- /dev/null +++ b/scripts/demo_base.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# Base配置演示流程脚本 +# 用于演示店匠通用客户的搜索效果 + +set -e + +cd "$(dirname "$0")/.." +source /home/tw/miniconda3/etc/profile.d/conda.sh +conda activate searchengine + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN}Base配置演示流程${NC}" +echo -e "${GREEN}========================================${NC}" + +# 加载.env配置文件 +if [ -f .env ]; then + set -a + source .env + set +a +fi + +# 配置参数（从环境变量或默认值） +TENANT_ID=${1:-"1"} +DB_HOST=${DB_HOST:-"120.79.247.228"} +DB_PORT=${DB_PORT:-"3316"} +DB_DATABASE=${DB_DATABASE:-"saas"} +DB_USERNAME=${DB_USERNAME:-"saas"} +DB_PASSWORD=${DB_PASSWORD:-"P89cZHS5d7dFyc9R"} +ES_HOST=${ES_HOST:-"http://localhost:9200"} +API_PORT=${API_PORT:-"6002"} +FRONTEND_PORT=${FRONTEND_PORT:-"6003"} + +echo -e "\n${YELLOW}配置参数:${NC}" +echo " Tenant ID: $TENANT_ID" +echo " MySQL: $DB_HOST:$DB_PORT/$DB_DATABASE" +echo " Elasticsearch: $ES_HOST" +echo " API Port: $API_PORT" +echo " Frontend Port: $FRONTEND_PORT" + +# Step 1: 生成测试数据 +echo -e "\n${YELLOW}Step 1/5: 生成测试数据${NC}" +if [ ! -f "test_data_base.sql" ]; then + echo "生成100条SPU测试数据..." + python scripts/generate_test_data.py \ + --num-spus 100 \ + --tenant-id "$TENANT_ID" \ + --start-spu-id 1 \ + --start-sku-id 1 \ + --output test_data_base.sql + echo -e "${GREEN}✓ 测试数据已生成: test_data_base.sql${NC}" +else + echo -e "${YELLOW}⚠ 测试数据文件已存在，跳过生成${NC}" +fi + +# Step 2: 导入测试数据到MySQL +echo -e "\n${YELLOW}Step 2/5: 导入测试数据到MySQL${NC}" +if [ -z "$DB_PASSWORD" ]; then + echo -e "${RED}ERROR: DB_PASSWORD未设置，请检查.env文件或环境变量${NC}" + exit 1 +fi + +python scripts/import_test_data.py \ + --db-host "$DB_HOST" \ + --db-port "$DB_PORT" \ + --db-database "$DB_DATABASE" \ + --db-username "$DB_USERNAME" \ + --db-password "$DB_PASSWORD" \ + --sql-file test_data_base.sql \ + --tenant-id "$TENANT_ID" + +echo -e "${GREEN}✓ 测试数据已导入MySQL${NC}" + +# Step 3: 导入数据到Elasticsearch +echo -e "\n${YELLOW}Step 3/5: 导入数据到Elasticsearch${NC}" +python scripts/ingest_shoplazza.py \ + --db-host "$DB_HOST" \ + --db-port "$DB_PORT" \ + --db-database "$DB_DATABASE" \ + --db-username "$DB_USERNAME" \ + --db-password "$DB_PASSWORD" \ + --tenant-id "$TENANT_ID" \ + --config base \ + --es-host "$ES_HOST" \ + --recreate \ + --batch-size 500 + +echo -e "${GREEN}✓ 数据已导入Elasticsearch${NC}" + +# Step 4: 启动后端服务 +echo -e "\n${YELLOW}Step 4/5: 启动后端服务${NC}" +echo "后端服务将在后台运行..." + +# 创建logs目录 +mkdir -p logs + +# 启动后端（后台运行） +nohup python -m api.app \ + --host 0.0.0.0 \ + --port "$API_PORT" \ + --customer base \ + --es-host "$ES_HOST" > logs/backend_base.log 2>&1 & + +BACKEND_PID=$! +echo $BACKEND_PID > logs/backend_base.pid +echo -e "${GREEN}后端服务已启动 (PID: $BACKEND_PID)${NC}" +echo -e "${GREEN}日志文件: logs/backend_base.log${NC}" + +# 等待后端启动 +echo -e "${YELLOW}等待后端服务启动...${NC}" +MAX_RETRIES=15 +RETRY_COUNT=0 +BACKEND_READY=false + +while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + sleep 2 + if curl -s "http://localhost:$API_PORT/health" > /dev/null 2>&1; then + BACKEND_READY=true + break + fi + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo -e "${YELLOW} 等待中... ($RETRY_COUNT/$MAX_RETRIES)${NC}" +done + +if [ "$BACKEND_READY" = true ]; then + echo -e "${GREEN}✓ 后端服务运行正常${NC}" +else + echo -e "${YELLOW}⚠ 后端服务可能还在启动中，请稍后访问${NC}" +fi + +# Step 5: 启动前端服务 +echo -e "\n${YELLOW}Step 5/5: 启动前端服务${NC}" +echo "前端服务将在后台运行..." + +# 创建base配置的前端JS文件 +echo "创建base配置前端文件..." +python scripts/create_base_frontend.py --tenant-id "$TENANT_ID" --api-port "$API_PORT" +echo -e "${GREEN}✓ 前端文件已创建${NC}" + +# 启动前端（后台运行） +export PORT="$FRONTEND_PORT" +nohup python scripts/frontend_server.py > logs/frontend_base.log 2>&1 & + +FRONTEND_PID=$! +echo $FRONTEND_PID > logs/frontend_base.pid +echo -e "${GREEN}前端服务已启动 (PID: $FRONTEND_PID)${NC}" +echo -e "${GREEN}日志文件: logs/frontend_base.log${NC}" + +# 等待前端启动 +echo -e "${YELLOW}等待前端服务启动...${NC}" +MAX_RETRIES=10 +RETRY_COUNT=0 +FRONTEND_READY=false + +while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + sleep 2 + if curl -s "http://localhost:$FRONTEND_PORT/" > /dev/null 2>&1; then + FRONTEND_READY=true + break + fi + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo -e "${YELLOW} 等待中... ($RETRY_COUNT/$MAX_RETRIES)${NC}" +done + +if [ "$FRONTEND_READY" = true ]; then + echo -e "${GREEN}✓ 前端服务运行正常${NC}" +else + echo -e "${YELLOW}⚠ 前端服务可能还在启动中，请稍后访问${NC}" +fi + +echo -e "\n${GREEN}========================================${NC}" +echo -e "${GREEN}演示环境启动完成!${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" +echo -e "访问地址:" +echo -e " ${GREEN}前端界面: http://localhost:$FRONTEND_PORT/base${NC}" +echo -e " ${GREEN}后端API: http://localhost:$API_PORT${NC}" +echo -e " ${GREEN}API文档: http://localhost:$API_PORT/docs${NC}" +echo "" +echo -e "配置信息:" +echo -e " Tenant ID: $TENANT_ID" +echo -e " Customer Config: base" +echo "" +echo -e "日志文件:" +echo -e " 后端: logs/backend_base.log" +echo -e " 前端: logs/frontend_base.log" +echo "" +echo -e "停止服务:" +echo -e " 所有服务: ./scripts/stop_base.sh" +echo -e " 单独停止后端: kill \$(cat logs/backend_base.pid)" +echo -e " 单独停止前端: kill \$(cat logs/frontend_base.pid)" +echo "" + diff --git a/scripts/frontend_server.py b/scripts/frontend_server.py index 1ef4cc4..02c1cec 100755 --- a/scripts/frontend_server.py +++ b/scripts/frontend_server.py @@ -15,7 +15,8 @@ from collections import defaultdict, deque frontend_dir = os.path.join(os.path.dirname(__file__), '../frontend') os.chdir(frontend_dir) -PORT = 6003 +# Get port from environment variable or default +PORT = int(os.getenv('PORT', 6003)) # Configure logging to suppress scanner noise logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s') @@ -44,6 +45,16 @@ class RateLimitingMixin: class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler, RateLimitingMixin): """Custom request handler with CORS support and robust error handling.""" + def do_GET(self): + """Handle GET requests with support for base.html.""" + # Route /base to base.html + if self.path == '/base' or self.path == '/base/': + self.path = '/base.html' + # Route / to index.html (default) + elif self.path == '/': + self.path = '/index.html' + return super().do_GET() + def setup(self): """Setup with error handling.""" try: diff --git a/scripts/stop_base.sh b/scripts/stop_base.sh new file mode 100755 index 0000000..2a663e0 --- /dev/null +++ b/scripts/stop_base.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Stop Base配置演示服务 + +set -e + +cd "$(dirname "$0")/.." + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +echo -e "${YELLOW}停止Base配置演示服务...${NC}" + +# Stop backend +if [ -f logs/backend_base.pid ]; then + BACKEND_PID=$(cat logs/backend_base.pid) + if ps -p $BACKEND_PID > /dev/null 2>&1; then + kill $BACKEND_PID + echo -e "${GREEN}✓ 后端服务已停止 (PID: $BACKEND_PID)${NC}" + else + echo -e "${YELLOW}⚠ 后端服务进程不存在${NC}" + fi + rm -f logs/backend_base.pid +else + echo -e "${YELLOW}⚠ 后端服务PID文件不存在${NC}" +fi + +# Stop frontend +if [ -f logs/frontend_base.pid ]; then + FRONTEND_PID=$(cat logs/frontend_base.pid) + if ps -p $FRONTEND_PID > /dev/null 2>&1; then + kill $FRONTEND_PID + echo -e "${GREEN}✓ 前端服务已停止 (PID: $FRONTEND_PID)${NC}" + else + echo -e "${YELLOW}⚠ 前端服务进程不存在${NC}" + fi + rm -f logs/frontend_base.pid +else + echo -e "${YELLOW}⚠ 前端服务PID文件不存在${NC}" +fi + +echo -e "${GREEN}所有服务已停止${NC}" + -- libgit2 0.21.2