L2添加耗时日志

Administrator

2024-04-24 9b7a928ead75ab099f8cc87a0f86f6870df7c9f7

 utils/data_export_util.py

@@ -14,8 +14,12 @@
from l2.huaxin import l2_huaxin_util
from log_module import log, log_export
from l2 import l2_data_source_util
from log_module.log import logger_debug
from trade import deal_big_money_manager
from utils import tool

# 缓存L2数据，格式：{"日期":{数据}}
__l2_data_cache = {}


def export_l2_excel(code, date=None):
@@ -25,19 +29,30 @@


# 获取L2的数据
def get_l2_datas(code, today_datas=None, date=None):
def get_l2_datas(code, today_datas=None, date=None, max_time=None):
    __start_time = time.time()
    if date is None:
        date = tool.get_now_date_str()
    datas = today_datas
    if datas is None:
        local_today_datas = log_export.load_l2_from_log(date)
        if date in __l2_data_cache:
            local_today_datas = __l2_data_cache.get(date)
        else:
            local_today_datas = log_export.load_l2_from_log(date)
            __l2_data_cache[date] = local_today_datas
        datas = local_today_datas.get(code)
    if not datas:
        datas = []
    logger_debug.info("加载L2逐笔委托耗时：{}", f"{(time.time() - __start_time)}")
    __start_time = time.time()
    process_indexs = log_export.get_l2_process_position(code, date)
    logger_debug.info("加载L2逐笔委托批次耗时：{}", f"{(time.time() - __start_time)}")
    __start_time = time.time()
    trade_indexs = log_export.get_l2_trade_position(code, date)
    real_position_indexes = log_export.get_real_place_order_positions(code, date)
    deal_list = log_export.load_huaxin_deal_record(code, date)
    logger_debug.info("加载其他交易参数耗时：{}", f"{(time.time() - __start_time)}")
    __start_time = time.time()
    deal_list_dict = {}
    for d in deal_list:
        deal_list_dict[str(d[0])] = d
@@ -46,9 +61,14 @@
    sell_nos = sell_no_dict.get(code)
    active_sell_map = log_export.load_huaxin_active_sell_map(date=date)
    active_sell_set = active_sell_map.get(code)
    logger_debug.info("加载卖单耗时：{}", f"{(time.time() - __start_time)}")
    __start_time = time.time()
    if not active_sell_set:
        active_sell_set = set()
    fdatas = export_l2_data(code, datas, process_indexs, trade_indexs, real_position_indexes, deal_list_dict, sell_nos, active_sell_set)
    fdatas = export_l2_data(code, datas, process_indexs, trade_indexs, real_position_indexes, deal_list_dict, sell_nos,
                            active_sell_set)
    logger_debug.info("组装数据耗时：{}", f"{(time.time() - __start_time)}")
    __start_time = time.time()
    return fdatas


@@ -61,7 +81,8 @@
    return fdatas


def export_l2_data(code, datas, process_indexs, trade_indexs, real_position_indexes, deal_list_dict, sell_nos,active_sell_nos):
def export_l2_data(code, datas, process_indexs, trade_indexs, real_position_indexes, deal_list_dict, sell_nos,
                   active_sell_nos):
    def find_process_index(index):
        for i in range(0, len(process_indexs)):
            if process_indexs[i][0] <= index <= process_indexs[i][1]:
@@ -74,30 +95,40 @@
                return trade_indexs[i]
        return None

    logger_debug.info("准备加载L2Map数据-----")
    # 数据预处理
    num_operate_map = {}
    l2.l2_data_util.load_num_operate_map(num_operate_map, code, datas)
    logger_debug.info("load_num_operate_map")

    buy_no_map = {}
    sell_no_map = {}
    l2.l2_data_util.load_buy_no_map(buy_no_map, code, datas)
    logger_debug.info("load_buy_no_map")
    l2.l2_data_util.load_sell_no_map(sell_no_map, code, datas)

    logger_debug.info("load_sell_no_map")
    l2.l2_data_util.load_canceled_buy_no_map(l2.l2_data_util.local_today_canceled_buyno_map, code, datas)

    logger_debug.info("load_canceled_buy_no_map")
    # num_dict = {}
    # for data in datas:
    #     if data["val"]["num"] not in num_dict:
    #         num_dict[data["val"]["num"]] = []
    #     num_dict[data["val"]["num"]].append(data)

    logger_debug.info(f"循环组装数据开始：长度-{len(datas)} 主动卖长度-{len(active_sell_nos)}")
    index = 0
    fdatas = []
    # 数据太多就需要过滤掉小金额
    is_data_too_large = len(datas) > 20000
    for data in datas:
        index += 1
        if is_data_too_large and data['val']['num'] * float(data['val']['price']) < 5000:
            if data["index"] not in real_position_indexes:
                continue

        # 先移除
        # if data['val']['orderNo'] in active_sell_nos:
        #     # 过滤主动卖
        #     continue
        if data['val']['orderNo'] in active_sell_nos:
            # 过滤主动卖
            continue

        trade_info = find_trade_index(data["index"])
        if not trade_info:
@@ -188,8 +219,8 @@
                    logging.exception(e)
            else:
                deal_info = deal_list_dict.get(str(data["val"].get("orderNo")))
                if deal_info:
                    cancel_info = l2_huaxin_util.convert_time(deal_info[3],
                if deal_info and len(deal_info) >= 5:
                    cancel_info = l2_huaxin_util.convert_time(deal_info[4],
                                                              with_ms=True)
        format_data.append(cancel_info)
        cancel_order_info = None
@@ -206,39 +237,7 @@
        format_data.append(cancel_order_info)
        format_data.append(data["val"].get("orderNo"))
        fdatas.append((style_int, trade_info, format_data))

    # 将订单号索引
    order_no_index_map = {}
    for i in range(len(fdatas)):
        d = fdatas[i][2]
        if d[6].find('撤') >= 0:
            continue
        order_no_index_map[int(d[10])] = i
    order_no_indexes = [(k, order_no_index_map[k]) for k in order_no_index_map]
    order_no_indexes.sort(key=lambda x: x[0])

    if sell_nos:
        for sell_info in sell_nos:
            if sell_info[1] * sell_info[2] < 50 * 10000:
                continue
            for i in range(len(order_no_indexes) - 1):
                if order_no_indexes[i][0] < sell_info[0] < order_no_indexes[i + 1][0]:
                    item = []
                    item.append(order_no_indexes[i + 1][1])
                    item.append(l2_huaxin_util.convert_time(sell_info[3][0], with_ms=True))
                    item.append("")
                    item.append(
                        "{}万".format(round(sell_info[1] * sell_info[2] / 10000, 1)))
                    item.append(sell_info[2])
                    item.append(sell_info[1] // 100)
                    item.append("主动卖")
                    item.append(1)
                    item.append(l2_huaxin_util.convert_time(sell_info[4][0], with_ms=True))
                    item.append(None)
                    item.append(sell_info[0])
                    fdatas.insert(order_no_indexes[i + 1][1], (0, None, item))
                    break

    logger_debug.info("循环组装数据完成")
    return fdatas