#!/bin/bash # Unified data ingestion script for SearchEngine # Ingests data from MySQL to Elasticsearch cd "$(dirname "$0")/.." source /home/tw/miniconda3/etc/profile.d/conda.sh conda activate searchengine GREEN='\033[0;32m' YELLOW='\033[1;33m' RED='\033[0;31m' NC='\033[0m' echo -e "${GREEN}========================================${NC}" echo -e "${GREEN}数据灌入脚本${NC}" echo -e "${GREEN}========================================${NC}" # Load config from .env file if it exists if [ -f .env ]; then set -a source .env set +a fi # Parameters TENANT_ID=${1:-""} RECREATE_INDEX=${2:-"false"} DB_HOST=${DB_HOST:-"120.79.247.228"} DB_PORT=${DB_PORT:-"3316"} DB_DATABASE=${DB_DATABASE:-"saas"} DB_USERNAME=${DB_USERNAME:-"saas"} DB_PASSWORD=${DB_PASSWORD:-"P89cZHS5d7dFyc9R"} ES_HOST=${ES_HOST:-"http://localhost:9200"} BATCH_SIZE=${BATCH_SIZE:-500} echo -e "\n${YELLOW}Configuration:${NC}" echo " Tenant ID: $TENANT_ID" echo " Recreate Index: $RECREATE_INDEX" echo " MySQL: $DB_HOST:$DB_PORT/$DB_DATABASE" echo " Elasticsearch: $ES_HOST" echo " Batch Size: $BATCH_SIZE" # Validate parameters if [ -z "$TENANT_ID" ]; then echo -e "${RED}ERROR: Tenant ID is required${NC}" echo "Usage: $0 [recreate_index]" echo " tenant_id: Required, tenant ID" echo " recreate_index: Optional, recreate index if exists (true/false, default: false)" exit 1 fi if [ -z "$DB_PASSWORD" ]; then echo -e "${RED}ERROR: DB_PASSWORD未设置,请检查.env文件或环境变量${NC}" exit 1 fi # Build command CMD="python scripts/ingest_shoplazza.py \ --db-host $DB_HOST \ --db-port $DB_PORT \ --db-database $DB_DATABASE \ --db-username $DB_USERNAME \ --db-password $DB_PASSWORD \ --tenant-id $TENANT_ID \ --es-host $ES_HOST \ --batch-size $BATCH_SIZE" if [ "$RECREATE_INDEX" = "true" ] || [ "$RECREATE_INDEX" = "1" ]; then CMD="$CMD --recreate" echo -e "\n${YELLOW}Warning: Index will be deleted and recreated!${NC}" fi echo -e "\n${YELLOW}Starting data ingestion...${NC}" eval $CMD if [ $? -eq 0 ]; then echo -e "\n${GREEN}========================================${NC}" echo -e "${GREEN}数据灌入完成!${NC}" echo -e "${GREEN}========================================${NC}" else echo -e "\n${RED}========================================${NC}" echo -e "${RED}数据灌入失败!${NC}" echo -e "${RED}========================================${NC}" exit 1 fi