Blame view

scripts/ingest.sh 2.42 KB
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
1
2
  #!/bin/bash
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
3
4
  # Unified data ingestion script for SearchEngine
  # Ingests data from MySQL to Elasticsearch
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
5
6
7
8
9
10
11
  
  cd "$(dirname "$0")/.."
  source /home/tw/miniconda3/etc/profile.d/conda.sh
  conda activate searchengine
  
  GREEN='\033[0;32m'
  YELLOW='\033[1;33m'
4d824a77   tangwang   所有租户共用一套统一配置.tena...
12
  RED='\033[0;31m'
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
13
14
15
  NC='\033[0m'
  
  echo -e "${GREEN}========================================${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
16
  echo -e "${GREEN}数据灌入脚本${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
17
18
  echo -e "${GREEN}========================================${NC}"
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
19
20
21
22
23
24
25
26
  # Load config from .env file if it exists
  if [ -f .env ]; then
      set -a
      source .env
      set +a
  fi
  
  # Parameters
41e1f8df   tangwang   店匠体系数据的搜索:mock da...
27
28
29
  TENANT_ID=${1:-""}
  RECREATE_INDEX=${2:-"false"}
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
30
31
32
33
34
35
36
  DB_HOST=${DB_HOST:-"120.79.247.228"}
  DB_PORT=${DB_PORT:-"3316"}
  DB_DATABASE=${DB_DATABASE:-"saas"}
  DB_USERNAME=${DB_USERNAME:-"saas"}
  DB_PASSWORD=${DB_PASSWORD:-"P89cZHS5d7dFyc9R"}
  ES_HOST=${ES_HOST:-"http://localhost:9200"}
  BATCH_SIZE=${BATCH_SIZE:-500}
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
37
38
  
  echo -e "\n${YELLOW}Configuration:${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
39
  echo "  Tenant ID: $TENANT_ID"
41e1f8df   tangwang   店匠体系数据的搜索:mock da...
40
  echo "  Recreate Index: $RECREATE_INDEX"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
41
42
43
  echo "  MySQL: $DB_HOST:$DB_PORT/$DB_DATABASE"
  echo "  Elasticsearch: $ES_HOST"
  echo "  Batch Size: $BATCH_SIZE"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
44
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
45
46
47
  # Validate parameters
  if [ -z "$TENANT_ID" ]; then
      echo -e "${RED}ERROR: Tenant ID is required${NC}"
41e1f8df   tangwang   店匠体系数据的搜索:mock da...
48
49
50
      echo "Usage: $0 <tenant_id> [recreate_index]"
      echo "  tenant_id: Required, tenant ID"
      echo "  recreate_index: Optional, recreate index if exists (true/false, default: false)"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
51
52
      exit 1
  fi
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
53
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
54
55
  if [ -z "$DB_PASSWORD" ]; then
      echo -e "${RED}ERROR: DB_PASSWORD未设置,请检查.env文件或环境变量${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
56
57
58
59
      exit 1
  fi
  
  # Build command
4d824a77   tangwang   所有租户共用一套统一配置.tena...
60
61
62
63
64
65
66
67
68
69
  CMD="python scripts/ingest_shoplazza.py \
      --db-host $DB_HOST \
      --db-port $DB_PORT \
      --db-database $DB_DATABASE \
      --db-username $DB_USERNAME \
      --db-password $DB_PASSWORD \
      --tenant-id $TENANT_ID \
      --es-host $ES_HOST \
      --batch-size $BATCH_SIZE"
  
41e1f8df   tangwang   店匠体系数据的搜索:mock da...
70
  if [ "$RECREATE_INDEX" = "true" ] || [ "$RECREATE_INDEX" = "1" ]; then
4d824a77   tangwang   所有租户共用一套统一配置.tena...
71
      CMD="$CMD --recreate"
41e1f8df   tangwang   店匠体系数据的搜索:mock da...
72
      echo -e "\n${YELLOW}Warning: Index will be deleted and recreated!${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
73
74
  fi
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
75
  echo -e "\n${YELLOW}Starting data ingestion...${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
76
77
  eval $CMD
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
78
79
80
81
82
83
84
85
86
87
  if [ $? -eq 0 ]; then
      echo -e "\n${GREEN}========================================${NC}"
      echo -e "${GREEN}数据灌入完成!${NC}"
      echo -e "${GREEN}========================================${NC}"
  else
      echo -e "\n${RED}========================================${NC}"
      echo -e "${RED}数据灌入失败!${NC}"
      echo -e "${RED}========================================${NC}"
      exit 1
  fi