deepwalk refactor for memsave and perfermance optimize

tangwang
1 parent 74330086
Showing 1 changed file with 8 additions and 3 deletions Show diff stats
offline_tasks/scripts/load_index_to_redis.py
@@ -42,11 +42,16 @@ def load_index_file(file_path, redis_client, key_prefix, expire_seconds=None):
                 continue
  
             parts = line.split('\t')
-            if len(parts) != 2:
-                logger.warning(f"Invalid line format: {line}")
+            if len(parts) < 2:
+                logger.warning(f"Invalid line format (expected at least 2 fields): {line}")
                 continue
  
-            key_suffix, value = parts
+            # 支持2字段和3字段格式
+            # 格式1 (2字段): item_id \t similar_items
+            # 格式2 (3字段): item_id \t item_name \t similar_items (推荐格式)
+            # 取第一个字段作为key，最后一个字段作为value
+            key_suffix = parts[0]
+            value = parts[-1]
             redis_key = f"{key_prefix}:{key_suffix}"
  
             # 存储到Redis