ingest.sh
2.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/bin/bash
# Unified data ingestion script for SearchEngine
# Ingests data from MySQL to Elasticsearch
cd "$(dirname "$0")/.."
source /home/tw/miniconda3/etc/profile.d/conda.sh
conda activate searchengine
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'
echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}数据灌入脚本${NC}"
echo -e "${GREEN}========================================${NC}"
# Load config from .env file if it exists
if [ -f .env ]; then
set -a
source .env
set +a
fi
# Parameters
TENANT_ID=${1:-"1"}
DB_HOST=${DB_HOST:-"120.79.247.228"}
DB_PORT=${DB_PORT:-"3316"}
DB_DATABASE=${DB_DATABASE:-"saas"}
DB_USERNAME=${DB_USERNAME:-"saas"}
DB_PASSWORD=${DB_PASSWORD:-"P89cZHS5d7dFyc9R"}
ES_HOST=${ES_HOST:-"http://localhost:9200"}
BATCH_SIZE=${BATCH_SIZE:-500}
RECREATE=${RECREATE:-false}
echo -e "\n${YELLOW}Configuration:${NC}"
echo " Tenant ID: $TENANT_ID"
echo " MySQL: $DB_HOST:$DB_PORT/$DB_DATABASE"
echo " Elasticsearch: $ES_HOST"
echo " Batch Size: $BATCH_SIZE"
echo " Recreate Index: $RECREATE"
# Validate parameters
if [ -z "$TENANT_ID" ]; then
echo -e "${RED}ERROR: Tenant ID is required${NC}"
echo "Usage: $0 <tenant_id> [batch_size] [recreate]"
exit 1
fi
if [ -z "$DB_PASSWORD" ]; then
echo -e "${RED}ERROR: DB_PASSWORD未设置，请检查.env文件或环境变量${NC}"
exit 1
fi
# Build command
CMD="python scripts/ingest_shoplazza.py \
--db-host $DB_HOST \
--db-port $DB_PORT \
--db-database $DB_DATABASE \
--db-username $DB_USERNAME \
--db-password $DB_PASSWORD \
--tenant-id $TENANT_ID \
--es-host $ES_HOST \
--batch-size $BATCH_SIZE"
if [ "$RECREATE" = "true" ] || [ "$RECREATE" = "1" ]; then
CMD="$CMD --recreate"
fi
echo -e "\n${YELLOW}Starting data ingestion...${NC}"
eval $CMD
if [ $? -eq 0 ]; then
echo -e "\n${GREEN}========================================${NC}"
echo -e "${GREEN}数据灌入完成!${NC}"
echo -e "${GREEN}========================================${NC}"
else
echo -e "\n${RED}========================================${NC}"
echo -e "${RED}数据灌入失败!${NC}"
echo -e "${RED}========================================${NC}"
exit 1
fi