#!/bin/bash
  

  cd /home/tw/recommendation/offline_tasks
  

  # 内存监控函数
  check_memory() {
      local pid=$1
      local threshold_warn=25  # 25GB警告阈值

      local threshold_kill=35  # 30GB强制kill阈值

      
      while kill -0 $pid 2>/dev/null; do
          # 获取进程内存使用（MB）
          local mem_mb=$(ps -p $pid -o rss= 2>/dev/null | awk '{print int($1/1024)}')
          
          if [ -n "$mem_mb" ]; then
              local mem_gb=$(echo "scale=2; $mem_mb/1024" | bc)
              local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
              
              if [ $(echo "$mem_gb >= $threshold_kill" | bc) -eq 1 ]; then
                  echo "[$timestamp] ❌ 内存超限！当前使用: ${mem_gb}GB (>= ${threshold_kill}GB), 强制终止进程 PID=$pid" | tee -a logs/memory_monitor.log
                  kill -9 $pid
                  break
              elif [ $(echo "$mem_gb >= $threshold_warn" | bc) -eq 1 ]; then
                  echo "[$timestamp] ⚠️  内存警告！当前使用: ${mem_gb}GB (>= ${threshold_warn}GB), PID=$pid" | tee -a logs/memory_monitor.log
              fi
          fi
          

          sleep 60

      done
  }
  

  # 清理旧进程
  ps -ef|grep run_all.py | awk '{print $2}' | xargs kill -9 2>/dev/null
  ps -ef|grep recommendation | awk '{print $2}' | xargs kill -9 2>/dev/null
  rm output/* -rf 2>/dev/null
  rm logs/* -rf 2>/dev/null
  mkdir -p logs

  echo "======================================================================"
  echo "开始运行离线任务 - $(date '+%Y-%m-%d %H:%M:%S')"
  echo "内存监控: 警告阈值=25GB, 强制终止阈值=30GB"
  echo "======================================================================"

  echo ""

  echo ">>>  run_all.py"

  # python3 run_all.py --lookback_days 400 --top_n 50 --debug &
  python3 run_all.py --debug &

  PID_PROD=$!
  echo "生产任务 PID: $PID_PROD"

  # 启动内存监控
  check_memory $PID_PROD &
  MONITOR_PID_2=$!
  
  # 等待生产任务完成
  wait $PID_PROD
  PROD_EXIT_CODE=$?
  kill $MONITOR_PID_2 2>/dev/null
  
  if [ $PROD_EXIT_CODE -eq 0 ]; then
      echo "✓ 生产模式完成"
  else
      echo "✗ 生产模式失败，退出码: $PROD_EXIT_CODE"
      exit 1
  fi

  echo ""
  echo ">>> 步骤3: 加载到Redis"

  python3 scripts/load_index_to_redis.py --redis-host localhost

  LOAD_EXIT_CODE=$?
  
  if [ $LOAD_EXIT_CODE -eq 0 ]; then
      echo "✓ Redis加载完成"
  else
      echo "✗ Redis加载失败，退出码: $LOAD_EXIT_CODE"
      exit 1
  fi
  
  echo ""
  echo "======================================================================"
  echo "所有任务完成 - $(date '+%Y-%m-%d %H:%M:%S')"
  echo "======================================================================"