Blame view

scripts/ingest.sh 2.2 KB
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
1
2
  #!/bin/bash
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
3
4
  # Unified data ingestion script for SearchEngine
  # Ingests data from MySQL to Elasticsearch
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
5
6
7
8
9
10
11
  
  cd "$(dirname "$0")/.."
  source /home/tw/miniconda3/etc/profile.d/conda.sh
  conda activate searchengine
  
  GREEN='\033[0;32m'
  YELLOW='\033[1;33m'
4d824a77   tangwang   所有租户共用一套统一配置.tena...
12
  RED='\033[0;31m'
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
13
14
15
  NC='\033[0m'
  
  echo -e "${GREEN}========================================${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
16
  echo -e "${GREEN}数据灌入脚本${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
17
18
  echo -e "${GREEN}========================================${NC}"
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
  # Load config from .env file if it exists
  if [ -f .env ]; then
      set -a
      source .env
      set +a
  fi
  
  # Parameters
  TENANT_ID=${1:-"1"}
  DB_HOST=${DB_HOST:-"120.79.247.228"}
  DB_PORT=${DB_PORT:-"3316"}
  DB_DATABASE=${DB_DATABASE:-"saas"}
  DB_USERNAME=${DB_USERNAME:-"saas"}
  DB_PASSWORD=${DB_PASSWORD:-"P89cZHS5d7dFyc9R"}
  ES_HOST=${ES_HOST:-"http://localhost:9200"}
  BATCH_SIZE=${BATCH_SIZE:-500}
  RECREATE=${RECREATE:-false}
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
36
37
  
  echo -e "\n${YELLOW}Configuration:${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
38
39
40
41
42
  echo "  Tenant ID: $TENANT_ID"
  echo "  MySQL: $DB_HOST:$DB_PORT/$DB_DATABASE"
  echo "  Elasticsearch: $ES_HOST"
  echo "  Batch Size: $BATCH_SIZE"
  echo "  Recreate Index: $RECREATE"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
43
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
44
45
46
47
48
49
  # Validate parameters
  if [ -z "$TENANT_ID" ]; then
      echo -e "${RED}ERROR: Tenant ID is required${NC}"
      echo "Usage: $0 <tenant_id> [batch_size] [recreate]"
      exit 1
  fi
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
50
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
51
52
  if [ -z "$DB_PASSWORD" ]; then
      echo -e "${RED}ERROR: DB_PASSWORD未设置,请检查.env文件或环境变量${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
53
54
55
56
      exit 1
  fi
  
  # Build command
4d824a77   tangwang   所有租户共用一套统一配置.tena...
57
58
59
60
61
62
63
64
65
66
67
68
  CMD="python scripts/ingest_shoplazza.py \
      --db-host $DB_HOST \
      --db-port $DB_PORT \
      --db-database $DB_DATABASE \
      --db-username $DB_USERNAME \
      --db-password $DB_PASSWORD \
      --tenant-id $TENANT_ID \
      --es-host $ES_HOST \
      --batch-size $BATCH_SIZE"
  
  if [ "$RECREATE" = "true" ] || [ "$RECREATE" = "1" ]; then
      CMD="$CMD --recreate"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
69
70
  fi
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
71
  echo -e "\n${YELLOW}Starting data ingestion...${NC}"
115047ee   tangwang   为一个租户灌入测试数据;实例的启动...
72
73
  eval $CMD
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
74
75
76
77
78
79
80
81
82
83
  if [ $? -eq 0 ]; then
      echo -e "\n${GREEN}========================================${NC}"
      echo -e "${GREEN}数据灌入完成!${NC}"
      echo -e "${GREEN}========================================${NC}"
  else
      echo -e "\n${RED}========================================${NC}"
      echo -e "${RED}数据灌入失败!${NC}"
      echo -e "${RED}========================================${NC}"
      exit 1
  fi