tangwang / recommendation

Commit be9c0900f74f3d2576edbe08a1e11581e5d626d1

Authored by tangwang
1 parent 8cf2517e

之前多个任务，配置分散，进行集中管理，全部统一到offline_tasks/config/offline_config.py 

因为在线服务跟搜索项目融合到一起，这里删除 config.py
Showing 7 changed files with 25 additions and 46 deletions   Show diff stats
Inline Side-by-side
config.py deleted
View file @8cf2517
... ... @@ -1,26 +0,0 @@
1   -import os
2   -
3   -
4   -ES_CONFIG = {
5   - 'host': 'http://localhost:9200',
6   - # default index name will be overwritten below based on APP_ENV
7   - 'index_name': 'spu',
8   - 'username': 'essa',
9   - 'password': '4hOaLaf41y2VuI8y'
10   -}
11   -
12   -
13   -# Redis Cache Configuration
14   -REDIS_CONFIG = {
15   - # 'host': '120.76.41.98',
16   - 'host': 'localhost',
17   - 'port': 6479,
18   - 'snapshot_db': 0,
19   - 'password': 'BMfv5aI31kgHWtlx',
20   - 'socket_timeout': 1,
21   - 'socket_connect_timeout': 1,
22   - 'retry_on_timeout': False,
23   - 'cache_expire_days': 180, # 6 months
24   - 'translation_cache_expire_days': 360,
25   - 'translation_cache_prefix': 'trans'
26   -}
offline_tasks/config/offline_config.py
  Show/Hide comments   View file @be9c090
... ... @@ -105,9 +105,9 @@ INTEREST_AGGREGATION_CONFIG = {
105 105 # Redis配置（用于存储索引）
106 106 REDIS_CONFIG = {
107 107 'host': 'localhost',
108   - 'port': 6379,
  108 + 'port': 6479, # 统一使用6479端口
109 109 'db': 0,
110   - 'password': None,
  110 + 'password': 'BMfv5aI31kgHWtlx', # 6479端口需要认证
111 111 'decode_responses': False
112 112 }
113 113  
... ...
offline_tasks/example_query_redis.py
  Show/Hide comments   View file @be9c090
... ... @@ -69,11 +69,11 @@ def query_interest_items(redis_client, dimension_key, list_type='hot', top_n=20)
69 69  
70 70 def main():
71 71 parser = argparse.ArgumentParser(description='Query recommendation results from Redis')
72   - parser.add_argument('--redis-host', type=str, default=REDIS_CONFIG.get('host', 'localhost'),
  72 + parser.add_argument('--redis-host', type=str, default=REDIS_CONFIG['host'],
73 73 help='Redis host')
74   - parser.add_argument('--redis-port', type=int, default=REDIS_CONFIG.get('port', 6379),
  74 + parser.add_argument('--redis-port', type=int, default=REDIS_CONFIG['port'],
75 75 help='Redis port')
76   - parser.add_argument('--redis-db', type=int, default=REDIS_CONFIG.get('db', 0),
  76 + parser.add_argument('--redis-db', type=int, default=REDIS_CONFIG['db'],
77 77 help='Redis database')
78 78  
79 79 args = parser.parse_args()
... ...
offline_tasks/run.sh
  Show/Hide comments   View file @be9c090
... ... @@ -30,8 +30,6 @@ TOP_N=50
30 30 DEBUG_MODE="--debug" # 留空则不开启debug
31 31  
32 32 # Redis配置
33   -REDIS_HOST="localhost"
34   -REDIS_PORT=6379
35 33  
36 34 # 内存监控阈值
37 35 MEM_WARN_THRESHOLD=25 # GB
... ... @@ -216,7 +214,7 @@ echo "======================================================================"
216 214 echo "[加载到Redis] 开始 - $(date '+%Y-%m-%d %H:%M:%S')"
217 215 echo "======================================================================"
218 216  
219   -python3 scripts/load_index_to_redis.py --redis-host $REDIS_HOST --redis-port $REDIS_PORT
  217 +python3 scripts/load_index_to_redis.py
220 218 LOAD_EXIT=$?
221 219  
222 220 if [ $LOAD_EXIT -eq 0 ]; then
... ...
offline_tasks/scripts/fix_redis_keys.py
  Show/Hide comments   View file @be9c090
... ... @@ -5,15 +5,19 @@
5 5 """
6 6 import redis
7 7 import sys
  8 +import os
  9 +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  10 +from config.offline_config import REDIS_CONFIG
8 11  
9 12 def fix_redis_keys():
10 13 """修复Redis key格式"""
11   -
  14 +
12 15 # 连接Redis
13 16 redis_client = redis.Redis(
14   - host='localhost',
15   - port=6379,
16   - db=0,
  17 + host=REDIS_CONFIG['host'],
  18 + port=REDIS_CONFIG['port'],
  19 + db=REDIS_CONFIG['db'],
  20 + password=REDIS_CONFIG['password'],
17 21 decode_responses=True
18 22 )
19 23  
... ...
offline_tasks/scripts/load_index_to_redis.py
  Show/Hide comments   View file @be9c090
... ... @@ -8,6 +8,9 @@ import logging
8 8 import os
9 9 import sys
10 10 from datetime import datetime
  11 +
  12 +# 添加父目录到Python路径
  13 +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
11 14 from config.offline_config import REDIS_CONFIG, OUTPUT_DIR
12 15  
13 16 def setup_logger():
... ... @@ -209,13 +212,13 @@ def load_interest_indices(redis_client, date_str=None, expire_days=7):
209 212  
210 213 def main():
211 214 parser = argparse.ArgumentParser(description='Load recommendation indices to Redis')
212   - parser.add_argument('--redis-host', type=str, default=REDIS_CONFIG.get('host', 'localhost'),
  215 + parser.add_argument('--redis-host', type=str, default=REDIS_CONFIG['host'],
213 216 help='Redis host')
214   - parser.add_argument('--redis-port', type=int, default=REDIS_CONFIG.get('port', 6379),
  217 + parser.add_argument('--redis-port', type=int, default=REDIS_CONFIG['port'],
215 218 help='Redis port')
216   - parser.add_argument('--redis-db', type=int, default=REDIS_CONFIG.get('db', 0),
  219 + parser.add_argument('--redis-db', type=int, default=REDIS_CONFIG['db'],
217 220 help='Redis database')
218   - parser.add_argument('--redis-password', type=str, default=REDIS_CONFIG.get('password'),
  221 + parser.add_argument('--redis-password', type=str, default=REDIS_CONFIG['password'],
219 222 help='Redis password')
220 223 parser.add_argument('--date', type=str, default=None,
221 224 help='Date string (YYYYMMDD), default is today')
... ...
offline_tasks/test_connection.py
  Show/Hide comments   View file @be9c090
... ... @@ -51,10 +51,10 @@ def test_redis_connection():
51 51  
52 52 try:
53 53 redis_client = redis.Redis(
54   - host=REDIS_CONFIG.get('host', 'localhost'),
55   - port=REDIS_CONFIG.get('port', 6379),
56   - db=REDIS_CONFIG.get('db', 0),
57   - password=REDIS_CONFIG.get('password'),
  54 + host=REDIS_CONFIG['host'],
  55 + port=REDIS_CONFIG['port'],
  56 + db=REDIS_CONFIG['db'],
  57 + password=REDIS_CONFIG['password'],
58 58 decode_responses=True
59 59 )
60 60  
... ...