并行从磁盘读取数据

Administrator

2025-07-03 752bd1e7ac94f9494f47f451fa227977bb42a22a

 utils/data_export_util.py

@@ -7,6 +7,7 @@
import os
import time

import dask
import xlwt

import constant
@@ -53,19 +54,29 @@
        real_position_indexes = []
        deal_list = []
        cancel_reasons = {}
        sell_no_dict = {}
        active_sell_map = {}
    else:
        process_indexs = log_export.get_l2_process_position(code, date)
        trade_indexs = log_export.get_l2_trade_position(code, date)
        real_position_indexes = log_export.get_real_place_order_positions(code, date)
        deal_list = log_export.load_huaxin_deal_record(code, date)
        cancel_reasons = log_export.load_cancel_buy_reasons(code, date)
        tasks = [dask.delayed(log_export.get_l2_process_position)(code, date),
                 dask.delayed(log_export.get_l2_trade_position)(code, date),
                 dask.delayed(log_export.get_real_place_order_positions)(code, date),
                 dask.delayed(log_export.load_huaxin_deal_record)(code, date),
                 dask.delayed(log_export.load_cancel_buy_reasons)(code, date),
                 dask.delayed(log_export.load_huaxin_transaction_sell_no)(code, date),
                 dask.delayed(log_export.load_huaxin_active_sell_map)(date),
                 ]
        results = dask.delayed(tasks).compute()
        trade_indexs = results[0]
        real_position_indexes = results[1]
        deal_list = results[2]
        cancel_reasons = results[3]
        process_indexs = results[4]
        sell_no_dict = results[5]
        active_sell_map = results[6]
    deal_list_dict = {}
    for d in deal_list:
        deal_list_dict[str(d[0])] = d

    sell_no_dict = log_export.load_huaxin_transaction_sell_no(code=code, date=date)
    sell_nos = sell_no_dict.get(code)
    active_sell_map = log_export.load_huaxin_active_sell_map(date=date)
    active_sell_set = active_sell_map.get(code)
    if not active_sell_set:
        active_sell_set = set()