From 752bd1e7ac94f9494f47f451fa227977bb42a22a Mon Sep 17 00:00:00 2001
From: Administrator <admin@example.com>
Date: 星期四, 03 七月 2025 18:07:37 +0800
Subject: [PATCH] 并行从磁盘读取数据
---
utils/data_export_util.py | 29 ++++++++++++++++++++---------
1 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/utils/data_export_util.py b/utils/data_export_util.py
index 4de83e7..bf06898 100644
--- a/utils/data_export_util.py
+++ b/utils/data_export_util.py
@@ -7,6 +7,7 @@
import os
import time
+import dask
import xlwt
import constant
@@ -53,19 +54,29 @@
real_position_indexes = []
deal_list = []
cancel_reasons = {}
+ sell_no_dict = {}
+ active_sell_map = {}
else:
- process_indexs = log_export.get_l2_process_position(code, date)
- trade_indexs = log_export.get_l2_trade_position(code, date)
- real_position_indexes = log_export.get_real_place_order_positions(code, date)
- deal_list = log_export.load_huaxin_deal_record(code, date)
- cancel_reasons = log_export.load_cancel_buy_reasons(code, date)
+ tasks = [dask.delayed(log_export.get_l2_process_position)(code, date),
+ dask.delayed(log_export.get_l2_trade_position)(code, date),
+ dask.delayed(log_export.get_real_place_order_positions)(code, date),
+ dask.delayed(log_export.load_huaxin_deal_record)(code, date),
+ dask.delayed(log_export.load_cancel_buy_reasons)(code, date),
+ dask.delayed(log_export.load_huaxin_transaction_sell_no)(code, date),
+ dask.delayed(log_export.load_huaxin_active_sell_map)(date),
+ ]
+ results = dask.delayed(tasks).compute()
+ trade_indexs = results[0]
+ real_position_indexes = results[1]
+ deal_list = results[2]
+ cancel_reasons = results[3]
+ process_indexs = results[4]
+ sell_no_dict = results[5]
+ active_sell_map = results[6]
deal_list_dict = {}
for d in deal_list:
deal_list_dict[str(d[0])] = d
-
- sell_no_dict = log_export.load_huaxin_transaction_sell_no(code=code, date=date)
sell_nos = sell_no_dict.get(code)
- active_sell_map = log_export.load_huaxin_active_sell_map(date=date)
active_sell_set = active_sell_map.get(code)
if not active_sell_set:
active_sell_set = set()
@@ -138,7 +149,7 @@
if limit_up_price:
# 闇�瑕佽闃呯殑鐗规畩鐨勯噺
special_volumes = BuyMoneyUtil.get_possible_buy_volumes(limit_up_price)
- special_volumes = [x//100 for x in special_volumes]
+ special_volumes = [x // 100 for x in special_volumes]
else:
special_volumes = []
--
Gitblit v1.8.0