并行从磁盘读取数据

Administrator

2025-07-03 752bd1e7ac94f9494f47f451fa227977bb42a22a

 utils/data_export_util.py

@@ -7,6 +7,7 @@
import os
import time

import dask
import xlwt

import constant
@@ -18,6 +19,7 @@
from l2 import l2_data_source_util, l2_data_util
from log_module.log import logger_debug
from trade import deal_big_money_manager
from trade.buy_money_count_setting import BuyMoneyUtil
from utils import tool

# 缓存L2数据，格式：{"日期":{数据}}
@@ -52,19 +54,29 @@
        real_position_indexes = []
        deal_list = []
        cancel_reasons = {}
        sell_no_dict = {}
        active_sell_map = {}
    else:
        process_indexs = log_export.get_l2_process_position(code, date)
        trade_indexs = log_export.get_l2_trade_position(code, date)
        real_position_indexes = log_export.get_real_place_order_positions(code, date)
        deal_list = log_export.load_huaxin_deal_record(code, date)
        cancel_reasons = log_export.load_cancel_buy_reasons(code, date)
        tasks = [dask.delayed(log_export.get_l2_process_position)(code, date),
                 dask.delayed(log_export.get_l2_trade_position)(code, date),
                 dask.delayed(log_export.get_real_place_order_positions)(code, date),
                 dask.delayed(log_export.load_huaxin_deal_record)(code, date),
                 dask.delayed(log_export.load_cancel_buy_reasons)(code, date),
                 dask.delayed(log_export.load_huaxin_transaction_sell_no)(code, date),
                 dask.delayed(log_export.load_huaxin_active_sell_map)(date),
                 ]
        results = dask.delayed(tasks).compute()
        trade_indexs = results[0]
        real_position_indexes = results[1]
        deal_list = results[2]
        cancel_reasons = results[3]
        process_indexs = results[4]
        sell_no_dict = results[5]
        active_sell_map = results[6]
    deal_list_dict = {}
    for d in deal_list:
        deal_list_dict[str(d[0])] = d

    sell_no_dict = log_export.load_huaxin_transaction_sell_no(code=code, date=date)
    sell_nos = sell_no_dict.get(code)
    active_sell_map = log_export.load_huaxin_active_sell_map(date=date)
    active_sell_set = active_sell_map.get(code)
    if not active_sell_set:
        active_sell_set = set()
@@ -132,10 +144,20 @@
    fdatas = []
    # 数据太多就需要过滤掉小金额
    is_data_too_large = len(datas) > 20000

    limit_up_price = gpcode_manager.get_limit_up_price_as_num(code)
    if limit_up_price:
        # 需要订阅的特殊的量
        special_volumes = BuyMoneyUtil.get_possible_buy_volumes(limit_up_price)
        special_volumes = [x // 100 for x in special_volumes]
    else:
        special_volumes = []

    for data in datas:
        index += 1
        if is_data_too_large and data['val']['num'] * float(data['val']['price']) < 5000:
            if data["index"] not in real_position_indexes:
            # 小单
            if data["index"] not in real_position_indexes and data['val']['num'] not in special_volumes:
                continue

        # 先移除
@@ -276,9 +298,10 @@
            format_data.append(f"{round(d[2] / 10000, 1)}万")
            format_data.append(round(d[2] / d[1], 2))
            format_data.append(d[1] // 100)
            format_data.append('买')
            format_data.append('主动买')
            format_data.append(1)
            format_data.append(l2_huaxin_util.convert_time(d[4], with_ms=True))
            format_data.append('')
            format_data.append(d[0])
            fdatas.insert(i, (0, None, format_data))
    logger_debug.info("循环组装数据完成")