Blame view

scripts/mock_data.sh 4.16 KB
4d824a77   tangwang   所有租户共用一套统一配置.tena...
1
2
3
4
  #!/bin/bash
  
  # Mock data script for SearchEngine
  # Generates test data and imports to MySQL
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
5
  # Supports both mock data generation and CSV import
4d824a77   tangwang   所有租户共用一套统一配置.tena...
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
  
  cd "$(dirname "$0")/.."
  source /home/tw/miniconda3/etc/profile.d/conda.sh
  conda activate searchengine
  
  GREEN='\033[0;32m'
  YELLOW='\033[1;33m'
  RED='\033[0;31m'
  NC='\033[0m'
  
  echo -e "${GREEN}========================================${NC}"
  echo -e "${GREEN}Mock Data Script${NC}"
  echo -e "${GREEN}========================================${NC}"
  
  # Load config from .env file if it exists
  if [ -f .env ]; then
      set -a
      source .env
      set +a
  fi
  
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  # Parse arguments
  MODE="mock"  # "mock" or "csv"
  TENANT_ID=""
  NUM_SPUS=100
  CSV_FILE=""
  START_SPU_ID=1
  
  # Parse command line arguments
  while [[ $# -gt 0 ]]; do
      case $1 in
          --mode)
              MODE="$2"
              shift 2
              ;;
          --tenant-id)
              TENANT_ID="$2"
              shift 2
              ;;
          --num-spus)
              NUM_SPUS="$2"
              shift 2
              ;;
          --csv-file)
              CSV_FILE="$2"
              shift 2
              ;;
          --start-spu-id)
              START_SPU_ID="$2"
              shift 2
              ;;
          *)
              # Legacy support: first arg as tenant_id, second as num_spus
              if [ -z "$TENANT_ID" ]; then
                  TENANT_ID="$1"
              elif [ "$NUM_SPUS" = "100" ] && [[ "$1" =~ ^[0-9]+$ ]]; then
                  NUM_SPUS="$1"
              fi
              shift
              ;;
      esac
  done
  
  # Set default tenant_id based on mode
  if [ -z "$TENANT_ID" ]; then
      if [ "$MODE" = "csv" ]; then
          TENANT_ID="2"
      else
          TENANT_ID="1"
      fi
  fi
  
  # Database configuration
4d824a77   tangwang   所有租户共用一套统一配置.tena...
79
80
81
82
83
  DB_HOST=${DB_HOST:-"120.79.247.228"}
  DB_PORT=${DB_PORT:-"3316"}
  DB_DATABASE=${DB_DATABASE:-"saas"}
  DB_USERNAME=${DB_USERNAME:-"saas"}
  DB_PASSWORD=${DB_PASSWORD:-"P89cZHS5d7dFyc9R"}
4d824a77   tangwang   所有租户共用一套统一配置.tena...
84
85
  
  echo -e "\n${YELLOW}Configuration:${NC}"
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
86
  echo "  Mode: $MODE"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
87
  echo "  Tenant ID: $TENANT_ID"
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
88
89
90
91
92
93
94
95
  if [ "$MODE" = "mock" ]; then
      echo "  Number of SPUs: $NUM_SPUS"
      SQL_FILE="test_data.sql"
  else
      echo "  CSV File: $CSV_FILE"
      echo "  Start SPU ID: $START_SPU_ID"
      SQL_FILE="customer1_data.sql"
  fi
4d824a77   tangwang   所有租户共用一套统一配置.tena...
96
97
98
  echo "  MySQL: $DB_HOST:$DB_PORT/$DB_DATABASE"
  echo "  SQL File: $SQL_FILE"
  
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
99
100
101
102
103
104
105
106
107
108
109
110
111
  # Validate CSV mode
  if [ "$MODE" = "csv" ]; then
      if [ -z "$CSV_FILE" ]; then
          echo -e "${RED}ERROR: CSV file is required in csv mode${NC}"
          echo "Usage: $0 --mode csv --csv-file <path> [--tenant-id <id>] [--start-spu-id <id>]"
          exit 1
      fi
      if [ ! -f "$CSV_FILE" ]; then
          echo -e "${RED}ERROR: CSV file not found: $CSV_FILE${NC}"
          exit 1
      fi
  fi
  
4d824a77   tangwang   所有租户共用一套统一配置.tena...
112
  # Step 1: Generate test data
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
  if [ "$MODE" = "mock" ]; then
      echo -e "\n${YELLOW}Step 1/2: 生成Mock测试数据${NC}"
      python scripts/generate_test_data.py \
          --num-spus $NUM_SPUS \
          --tenant-id "$TENANT_ID" \
          --start-spu-id 1 \
          --start-sku-id 1 \
          --output "$SQL_FILE"
  else
      echo -e "\n${YELLOW}Step 1/2: 从CSV生成数据${NC}"
      python scripts/import_customer1_csv.py \
          --csv-file "$CSV_FILE" \
          --tenant-id "$TENANT_ID" \
          --start-spu-id $START_SPU_ID \
          --output "$SQL_FILE"
  fi
4d824a77   tangwang   所有租户共用一套统一配置.tena...
129
130
  
  if [ $? -ne 0 ]; then
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
131
      echo -e "${RED}✗ 生成数据失败${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
132
133
134
      exit 1
  fi
  
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
135
  echo -e "${GREEN}✓ 数据已生成: $SQL_FILE${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
136
  
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
137
138
  # Step 2: Import data to MySQL
  echo -e "\n${YELLOW}Step 2/2: 导入数据到MySQL${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
  if [ -z "$DB_PASSWORD" ]; then
      echo -e "${RED}ERROR: DB_PASSWORD未设置,请检查.env文件或环境变量${NC}"
      exit 1
  fi
  
  python scripts/import_test_data.py \
      --db-host "$DB_HOST" \
      --db-port "$DB_PORT" \
      --db-database "$DB_DATABASE" \
      --db-username "$DB_USERNAME" \
      --db-password "$DB_PASSWORD" \
      --sql-file "$SQL_FILE" \
      --tenant-id "$TENANT_ID"
  
  if [ $? -ne 0 ]; then
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
154
      echo -e "${RED}✗ 导入数据失败${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
155
156
157
      exit 1
  fi
  
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
158
  echo -e "${GREEN}✓ 数据已导入MySQL${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
159
160
  
  echo -e "\n${GREEN}========================================${NC}"
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
161
  echo -e "${GREEN}数据导入完成!${NC}"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
162
163
164
  echo -e "${GREEN}========================================${NC}"
  echo ""
  echo -e "下一步:"
a5a3856d   tangwang   店匠体系数据的搜索:mock da...
165
  echo -e "  ${YELLOW}./scripts/ingest.sh $TENANT_ID${NC}  - 从MySQL灌入数据到ES"
4d824a77   tangwang   所有租户共用一套统一配置.tena...
166
  echo ""