From 752bd1e7ac94f9494f47f451fa227977bb42a22a Mon Sep 17 00:00:00 2001
From: Administrator <admin@example.com>
Date: 星期四, 03 七月 2025 18:07:37 +0800
Subject: [PATCH] 并行从磁盘读取数据
---
utils/data_export_util.py | 40 +++++++++++++++++++++++++++++++---------
1 files changed, 31 insertions(+), 9 deletions(-)
diff --git a/utils/data_export_util.py b/utils/data_export_util.py
index cb6f849..bf06898 100644
--- a/utils/data_export_util.py
+++ b/utils/data_export_util.py
@@ -7,6 +7,7 @@
import os
import time
+import dask
import xlwt
import constant
@@ -18,6 +19,7 @@
from l2 import l2_data_source_util, l2_data_util
from log_module.log import logger_debug
from trade import deal_big_money_manager
+from trade.buy_money_count_setting import BuyMoneyUtil
from utils import tool
# 缂撳瓨L2鏁版嵁锛屾牸寮忥細{"鏃ユ湡":{鏁版嵁}}
@@ -52,19 +54,29 @@
real_position_indexes = []
deal_list = []
cancel_reasons = {}
+ sell_no_dict = {}
+ active_sell_map = {}
else:
- process_indexs = log_export.get_l2_process_position(code, date)
- trade_indexs = log_export.get_l2_trade_position(code, date)
- real_position_indexes = log_export.get_real_place_order_positions(code, date)
- deal_list = log_export.load_huaxin_deal_record(code, date)
- cancel_reasons = log_export.load_cancel_buy_reasons(code, date)
+ tasks = [dask.delayed(log_export.get_l2_process_position)(code, date),
+ dask.delayed(log_export.get_l2_trade_position)(code, date),
+ dask.delayed(log_export.get_real_place_order_positions)(code, date),
+ dask.delayed(log_export.load_huaxin_deal_record)(code, date),
+ dask.delayed(log_export.load_cancel_buy_reasons)(code, date),
+ dask.delayed(log_export.load_huaxin_transaction_sell_no)(code, date),
+ dask.delayed(log_export.load_huaxin_active_sell_map)(date),
+ ]
+ results = dask.delayed(tasks).compute()
+ trade_indexs = results[0]
+ real_position_indexes = results[1]
+ deal_list = results[2]
+ cancel_reasons = results[3]
+ process_indexs = results[4]
+ sell_no_dict = results[5]
+ active_sell_map = results[6]
deal_list_dict = {}
for d in deal_list:
deal_list_dict[str(d[0])] = d
-
- sell_no_dict = log_export.load_huaxin_transaction_sell_no(code=code, date=date)
sell_nos = sell_no_dict.get(code)
- active_sell_map = log_export.load_huaxin_active_sell_map(date=date)
active_sell_set = active_sell_map.get(code)
if not active_sell_set:
active_sell_set = set()
@@ -132,10 +144,20 @@
fdatas = []
# 鏁版嵁澶灏遍渶瑕佽繃婊ゆ帀灏忛噾棰�
is_data_too_large = len(datas) > 20000
+
+ limit_up_price = gpcode_manager.get_limit_up_price_as_num(code)
+ if limit_up_price:
+ # 闇�瑕佽闃呯殑鐗规畩鐨勯噺
+ special_volumes = BuyMoneyUtil.get_possible_buy_volumes(limit_up_price)
+ special_volumes = [x // 100 for x in special_volumes]
+ else:
+ special_volumes = []
+
for data in datas:
index += 1
if is_data_too_large and data['val']['num'] * float(data['val']['price']) < 5000:
- if data["index"] not in real_position_indexes:
+ # 灏忓崟
+ if data["index"] not in real_position_indexes and data['val']['num'] not in special_volumes:
continue
# 鍏堢Щ闄�
--
Gitblit v1.8.0