add cpp swing for mem optimize

tangwang
1 parent a1c26d3d
Showing 17 changed files with 36 additions and 6 deletions Show diff stats
collaboration/.gitignore -> offline_tasks/collaboration/.gitignore
collaboration/Makefile -> offline_tasks/collaboration/Makefile
collaboration/README.md -> offline_tasks/collaboration/README.md
collaboration/Swing快速开始.md -> offline_tasks/collaboration/Swing快速开始.md
collaboration/bin/icf_simple -> offline_tasks/collaboration/bin/icf_simple
collaboration/bin/swing -> offline_tasks/collaboration/bin/swing
collaboration/bin/swing_symmetric -> offline_tasks/collaboration/bin/swing_symmetric
collaboration/eval.py -> offline_tasks/collaboration/eval.py
collaboration/include/BitMap.h -> offline_tasks/collaboration/include/BitMap.h
collaboration/include/utils.h -> offline_tasks/collaboration/include/utils.h
collaboration/run.sh -> offline_tasks/collaboration/run.sh
collaboration/src/icf_simple.cc -> offline_tasks/collaboration/src/icf_simple.cc
collaboration/src/swing.cc -> offline_tasks/collaboration/src/swing.cc
collaboration/src/swing_symmetric.cc -> offline_tasks/collaboration/src/swing_symmetric.cc
collaboration/src/ucf.py -> offline_tasks/collaboration/src/ucf.py
collaboration/utils/utils.cc -> offline_tasks/collaboration/utils/utils.cc
offline_tasks/doc/Redis数据规范.md
@@ -294,15 +294,21 @@ def load_interest_index(file_path, list_type, redis_client, expire_seconds=25920
 ```bash
 cd /home/tw/recommendation/offline_tasks
-# 加载所有索引（使用今天的数据）
+# 加载所有索引（使用今天的数据，包括C++ Swing）
 python3 scripts/load_index_to_redis.py --redis-host localhost --redis-port 6379
 # 加载指定日期的索引
 python3 scripts/load_index_to_redis.py --date 20251016 --redis-host localhost
-# 只加载i2i索引
+# 只加载i2i索引（包括C++ Swing）
 python3 scripts/load_index_to_redis.py --load-i2i --redis-host localhost
+# 只加载C++ Swing索引
+python3 scripts/load_index_to_redis.py \
+    --file ../collaboration/output/swing_similar.txt \
+    --algorithm swing_cpp \
+    --redis-host localhost
+
 # 只加载兴趣聚合索引
 python3 scripts/load_index_to_redis.py --load-interest --redis-host localhost
 ```
@@ -315,7 +321,10 @@ redis-cli
 # 检查key数量
 DBSIZE
-# 查看某个商品的相似推荐
+# 查看某个商品的相似推荐（C++ Swing）
+GET item:similar:swing_cpp:3600052
+
+# 查看某个商品的相似推荐（Python Swing）
 GET item:similar:swing:12345
 # 查看平台热门商品
@@ -324,10 +333,17 @@ GET interest:hot:platform:pc
 # 查看所有i2i相关的key
 KEYS item:similar:*
+# 查看C++ Swing的key
+KEYS item:similar:swing_cpp:*
+
+# 查看Python Swing的key
+KEYS item:similar:swing:*
+
 # 查看所有interest相关的key
 KEYS interest:*
 # 检查key的过期时间
+TTL item:similar:swing_cpp:3600052
 TTL item:similar:swing:12345
 ```
@@ -337,6 +353,7 @@ TTL item:similar:swing:12345
 | 索引类型 | Key数量 | 单条Value大小 | 总内存 |
 |---------|--------|-------------|--------|
+| i2i_swing_cpp | 50,000 | ~400B | ~20MB |
 | i2i_swing | 50,000 | ~500B | ~25MB |
 | i2i_w2v | 50,000 | ~500B | ~25MB |
 | i2i_deepwalk | 50,000 | ~500B | ~25MB |
@@ -346,7 +363,12 @@ TTL item:similar:swing:12345
 | interest_cart | 10,000 | ~1KB | ~10MB |
 | interest_new | 5,000 | ~1KB | ~5MB |
 | interest_global | 10,000 | ~1KB | ~10MB |
-| **总计** | **270,000** | - | **~160MB** |
+| **总计** | **320,000** | - | **~180MB** |
+
+**说明**:
+- C++ Swing数据更紧凑（无商品名），单条大小约400B
+- 建议生产环境使用C++ Swing (`swing_cpp`)，性能更优
+- Python Swing可作为对照组或特殊场景使用
 ### 过期策略
@@ -373,17 +395,25 @@ TTL item:similar:swing:12345
 ```python
 # 检查加载成功率
 total_keys = redis_client.dbsize()
-expected_keys = 245000
+expected_keys = 320000  # 更新：包含C++ Swing
 success_rate = total_keys / expected_keys * 100
 # 检查数据完整性
 sample_keys = [
-    'item:similar:swing:12345',
+    'item:similar:swing_cpp:3600052',  # C++ Swing
+    'item:similar:swing:12345',        # Python Swing
+    'item:similar:w2v:12345',
     'interest:hot:platform:pc'
 ]
 for key in sample_keys:
     if not redis_client.exists(key):
         print(f"Missing key: {key}")
+
+# 检查C++ Swing vs Python Swing覆盖率
+cpp_swing_count = len(redis_client.keys('item:similar:swing_cpp:*'))
+py_swing_count = len(redis_client.keys('item:similar:swing:*'))
+print(f"C++ Swing keys: {cpp_swing_count}")
+print(f"Python Swing keys: {py_swing_count}")
 ```
 ### 性能指标