From cc2c7a8514d3d6eea4f73b5875c2500883d165f0 Mon Sep 17 00:00:00 2001
From: Administrator <admin@example.com>
Date: 星期一, 26 五月 2025 22:57:20 +0800
Subject: [PATCH] 增加自动加白接口
---
l2_data_parser.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++-----
1 files changed, 93 insertions(+), 10 deletions(-)
diff --git a/l2_data_parser.py b/l2_data_parser.py
index 8018fc5..0574cfb 100644
--- a/l2_data_parser.py
+++ b/l2_data_parser.py
@@ -5,8 +5,11 @@
import os
import sys
import time
+from multiprocessing import Pool
+
import pandas as pd
+from data_parser import transaction_big_order_parser
from db import mysql_data_delegate as mysql_data
from huaxin_client.l2_client_test import L2TransactionDataManager
from log_module import log_export
@@ -224,8 +227,11 @@
# 灏嗘枃浠跺啓鍏ュ埌鏂囨湰
writer.writerow(row)
+def test1(args):
+ index, df = args
+ print(index)
-def test(csv_path="E:/娴嬭瘯鏁版嵁/Transaction_Test.csv"):
+def pre_process_transactions(csv_path="E:/娴嬭瘯鏁版嵁/Transaction_Test.csv"):
def str_to_float(s):
try:
# 绉婚櫎鍗曚綅骞惰浆鎹�
@@ -260,19 +266,31 @@
'EndPrice': group['TradePrice'].iloc[-1]
})
+
+
+
dtype = {
'SecurityID': 'category', # 浣庡熀鏁板垎绫绘暟鎹�
}
- chunk_size = 100000
+ chunk_size = 10000
# 鍒涘缓DataFrame
chunks = pd.read_csv(csv_path, chunksize=chunk_size)
+ indexed_data = list(enumerate(chunks))
+ # 鏂板啓娉�
+ with Pool(processes=4) as pool:
+ pool.map(test1, indexed_data)
+
result_list = []
- index = 0
- for df in chunks:
- index += 1
+ for chunk_index, chunk in enumerate(chunks):
+ df = chunk.copy()
+ index = chunk_index + 1
+ child_path = csv_path.replace(".csv", f"_{index}.csv")
+ if os.path.exists(child_path):
+ continue
print(f"澶勭悊绗瑊index}鎵规")
df["TradePrice"] = df["TradePrice"].apply(str_to_float)
df["SecurityID"] = df["SecurityID"].apply(code_format)
+ df = df[df["SecurityID"].str.startswith(("30", "00", "60"), na=False)]
# 璁＄畻鎴愪氦閲戦
df['TradeAmount'] = df['TradePrice'] * df['TradeVolume']
@@ -281,18 +299,73 @@
# 搴旂敤鑱氬悎鍑芥暟
chunk_result = grouped.apply(first_last).reset_index()
- child_path = csv_path.replace(".csv", f"_{index}.csv")
+
chunk_result.to_csv(child_path, index=False)
print(f"澶勭悊瀹屾瘯锛屾�诲叡{index}鎵�")
-if __name__ == '__main__':
+def pre_process_ngtstick(csv_path="E:/娴嬭瘯鏁版嵁/NGTSTick_Test.csv"):
+ def str_to_float(s):
+ try:
+ # 绉婚櫎鍗曚綅骞惰浆鎹�
+ return round(float(s.split("@")[0]), 2)
+ except:
+ return float("nan")
+
+ def code_format(s):
+ try:
+ code = "{0:0>6}".format(s)
+ return code
+ except:
+ return ''
+
+ # 瀹氫箟鑱氬悎鍑芥暟
+ def first_last(group):
+ return pd.Series({
+ 'TotalAmount': group['TradeMoney'].sum(),
+ 'TotalVolume': group['Volume'].sum(),
+ 'StartTime': group['TickTime'].iloc[0],
+ 'StartPrice': group['Price'].iloc[0],
+ 'EndTime': group['TickTime'].iloc[-1],
+ 'EndPrice': group['Price'].iloc[-1]
+ })
+
+ # [ExchangeID,SecurityID,MainSeq,SubSeq,TickTime,TickType,BuyNo,SellNo,Price,Volume,TradeMoney,Side,TradeBSFlag,MDSecurityStat,Info1,Info2,Info3,LocalTimeStamp]
+
+ chunk_size = 10000
+ # 鍒涘缓DataFrame
+ chunks = pd.read_csv(csv_path, chunksize=chunk_size)
+ result_list = []
+ index = 0
+ for df in chunks:
+ index += 1
+ child_path = csv_path.replace(".csv", f"_{index}.csv")
+ if os.path.exists(child_path):
+ continue
+ print(f"澶勭悊绗瑊index}鎵规")
+ df = df[df["TickType"] == 'T']
+ df["Price"] = df["Price"].apply(str_to_float)
+ df["SecurityID"] = df["SecurityID"].apply(code_format)
+
+ df = df[df["SecurityID"].str.startswith(("30", "00", "60"), na=False)]
+
+ # 璁＄畻鎴愪氦閲戦
+ df['TradeMoney'] = df["TradeMoney"].apply(str_to_float)
+ # 鎸塖ecurityID鍜孊uyNo鍒嗙粍
+ grouped = df.groupby(['SecurityID', 'BuyNo'])
+ # 搴旂敤鑱氬悎鍑芥暟
+ chunk_result = grouped.apply(first_last).reset_index()
+ chunk_result.to_csv(child_path, index=False)
+ print(f"澶勭悊瀹屾瘯锛屾�诲叡{index}鎵�")
+
+
+if __name__ == '__main__1':
# df = pd.read_csv(f"E:/娴嬭瘯鏁版嵁/Transaction_Test.csv")
- test()
+ pre_process_transactions()
# 鍛戒护妯″紡 /home/userzjj/app/gp-server/l2_data_parser Transaction 2025-05-08
# 瑙ｆ瀽澶у崟锛� /home/userzjj/app/gp-server/l2_data_parser ExtractDealBigOrder 2025-05-09 /home/userzjj/鏈�缁堟垚浜ゆ暟鎹�20250509.txt 000555
-if __name__ == '__main__1':
+if __name__ == '__main__':
if len(sys.argv) > 1:
params = sys.argv[1:]
print("鎺ユ敹鐨勫弬鏁�", params)
@@ -311,7 +384,17 @@
elif _type == 'MarketData':
parse_market_data(day)
elif _type == 'Transaction_New':
- test(f"/home/userzjj/ftp/{day}/Transaction.csv")
+ transaction_big_order_parser.pre_process_transactions(f"/home/userzjj/ftp/{day}/Transaction.csv")
+ transaction_big_order_parser.concat_pre_transactions(f"/home/userzjj/ftp/{day}/Transaction")
+ elif _type == 'NGTSTick_New':
+ transaction_big_order_parser.pre_process_ngtsticks(f"/home/userzjj/ftp/{day}/NGTSTick.csv")
+ transaction_big_order_parser.concat_pre_ngtsticks(f"/home/userzjj/ftp/{day}/NGTSTick")
+ elif _type == 'Transaction_Concat':
+ transaction_big_order_parser.concat_pre_transactions(f"/home/userzjj/ftp/{day}/Transaction")
+ elif _type == 'NGTSTick_Concat':
+ transaction_big_order_parser.concat_pre_ngtsticks(f"/home/userzjj/ftp/{day}/NGTSTick")
+
+
elif _type == 'ExtractDealBigOrder':
# 鎻愬彇鎵�鏈夋垚浜ょ殑澶у崟
if len(params) > 2:
--
Gitblit v1.8.0