From ec1d64c71f86ff2564e709a75fda9f487acc2610 Mon Sep 17 00:00:00 2001
From: Administrator <admin@example.com>
Date: 星期一, 27 五月 2024 10:06:18 +0800
Subject: [PATCH] bug修复
---
third_data/kpl_data_manager.py | 280 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
1 files changed, 251 insertions(+), 29 deletions(-)
diff --git a/third_data/kpl_data_manager.py b/third_data/kpl_data_manager.py
index ebcb14a..9130b94 100644
--- a/third_data/kpl_data_manager.py
+++ b/third_data/kpl_data_manager.py
@@ -1,19 +1,26 @@
+import copy
import json
+import logging
import os
+import threading
+import time
+
+import requests
import constant
+from db.redis_manager_delegate import RedisUtils
+from log_module import async_log_util, log
from utils import tool
# 寮�鐩樺暒鍘嗗彶娑ㄥ仠鏁版嵁绠＄悊
-from db import mysql_data, redis_manager
-from log_module.log import logger_kpl_limit_up_reason_change
+from db import mysql_data_delegate as mysql_data, redis_manager_delegate as redis_manager
+from log_module.log import logger_kpl_limit_up_reason_change, logger_debug, logger_kpl_limit_up, \
+ logger_kpl_open_limit_up
from third_data import kpl_util, kpl_api
from third_data.code_plate_key_manager import LimitUpCodesPlateKeyManager, CodesHisReasonAndBlocksManager
# 浠ｇ爜瀵瑰簲鐨勬定鍋滃師鍥犱繚瀛�
from third_data.kpl_util import KPLPlatManager, KPLDataType
-
-
class KPLCodeLimitUpReasonManager:
@@ -23,13 +30,13 @@
return self.__redisManager.getRedis()
def save_reason(self, code, reason):
- self.__get_redis().setex(f"kpl_limitup_reason-{code}", tool.get_expire(), reason)
+ RedisUtils.setex(self.__get_redis(), f"kpl_limitup_reason-{code}", tool.get_expire(), reason)
def list_all(self):
- keys = self.__get_redis().keys("kpl_limitup_reason-*")
+ keys = RedisUtils.keys(self.__get_redis(), "kpl_limitup_reason-*")
dict_ = {}
for k in keys:
- val = self.__get_redis().get(k)
+ val = RedisUtils.get(self.__get_redis(), k)
dict_[k.split("-")[1]] = val
return dict_
@@ -41,6 +48,13 @@
__kplPlatManager = KPLPlatManager()
__LimitUpCodesPlateKeyManager = LimitUpCodesPlateKeyManager()
__CodesPlateKeysManager = CodesHisReasonAndBlocksManager()
+ __current_code_reasons_dict = {}
+ # 褰撳墠娑ㄥ仠鍘熷洜+鎺ㄨ崘鍘熷洜鐨勪唬鐮侀泦鍚�
+ __current_reason_codes_dict = {}
+ # 褰撳墠娑ㄥ仠鍘熷洜鐨勪唬鐮侀泦鍚�
+ __current_limit_up_reason_codes_dict = {}
+ __records_cache = {}
+ record_code_dict = {}
@classmethod
def __load_hist_and_blocks(cls, code):
@@ -58,9 +72,24 @@
@classmethod
def save_record(cls, day, records):
+ # 缁熻鐐告澘
+ try:
+ last_codes = set()
+ if cls.latest_origin_datas:
+ last_codes = set([x[0] for x in cls.latest_origin_datas])
+ now_codes = set()
+ if records:
+ now_codes = set([x[0] for x in records])
+ open_limit_up_codes = last_codes - now_codes
+ if open_limit_up_codes:
+ logger_kpl_open_limit_up.info(f"鐐告澘浠ｇ爜锛歿open_limit_up_codes}")
+ except Exception as e:
+ pass
+
# 缁熻浠ｇ爜鎵�灞炴澘鍧�
code_block_dict = {}
for data in records:
+ cls.record_code_dict[data[0]] = data
blocks = set(data[5].split("銆�"))
code = data[0]
for b in blocks:
@@ -68,9 +97,32 @@
code_block_dict[code] = set()
code_block_dict[code].add(b)
# 璁剧疆娑ㄥ仠鏁版嵁
+
if records:
cls.latest_origin_datas = records
- cls.__LimitUpCodesPlateKeyManager.set_today_limit_up([(r[0], r[5]) for r in records])
+ cls.__LimitUpCodesPlateKeyManager.set_today_limit_up(
+ [(r[0], r[5], r[6].split('銆�') if r[6] else []) for r in records])
+
+ code_reasons_dict = {}
+ reason_codes_dict = {}
+ limit_up_reason_codes_dict = {}
+ for d in records:
+ if d[5] not in limit_up_reason_codes_dict:
+ limit_up_reason_codes_dict[d[5]] = set()
+ limit_up_reason_codes_dict[d[5]].add(d[0])
+
+ # 娑ㄥ仠鍘熷洜 + 鎺ㄨ崘鍘熷洜
+ bs = {d[5]}
+ if d[6]:
+ bs |= set(d[6].split("銆�"))
+ code_reasons_dict[d[0]] = bs
+ for b in bs:
+ if b not in reason_codes_dict:
+ reason_codes_dict[b] = set()
+ reason_codes_dict[b].add(d[0])
+ cls.__current_code_reasons_dict = code_reasons_dict
+ cls.__current_reason_codes_dict = reason_codes_dict
+ cls.__current_limit_up_reason_codes_dict = limit_up_reason_codes_dict
# 娑ㄥ仠鏁版嵁璁板綍
mysqldb = mysql_data.Mysqldb()
@@ -119,7 +171,8 @@
@classmethod
def load_total_datas(cls):
cls.total_datas = KPLLimitUpDataRecordManager.list_all(tool.get_now_date_str())
- cls.__LimitUpCodesPlateKeyManager.set_today_total_limit_up([(r[3], r[2]) for r in cls.total_datas])
+ cls.__LimitUpCodesPlateKeyManager.set_today_total_limit_up(
+ [(r[3], r[2], r[6].split("銆�") if r[6] else []) for r in cls.total_datas])
for d in cls.total_datas:
cls.__load_hist_and_blocks(d[3])
@@ -127,6 +180,15 @@
def list_all(day):
mysqldb = mysql_data.Mysqldb()
return mysqldb.select_all(f"select * from kpl_limit_up_record where _day='{day}'")
+
+ @classmethod
+ def list_all_cache(cls, day):
+ if day in cls.__records_cache:
+ return cls.__records_cache[day]
+ fdata = cls.list_all(day)
+ if fdata:
+ cls.__records_cache[day] = fdata
+ return fdata
@staticmethod
def list_by_code(code, day):
@@ -158,16 +220,20 @@
for b in constant.KPL_INVALID_BLOCKS:
wheres.append(f"hb.`_hot_block_name` != '{b}'")
wheres = " and ".join(wheres)
- sql = f"SELECT GROUP_CONCAT(_hot_block_name) FROM (SELECT hb.`_hot_block_name`,hb.`_day` FROM `kpl_limit_up_record` hb WHERE hb.`_code`='{code}' AND {wheres} ORDER BY hb.`_day` DESC LIMIT 10) a GROUP BY a._day ORDER BY a._day DESC LIMIT 1"
+ sql = f"SELECT GROUP_CONCAT(_hot_block_name) FROM (SELECT hb.`_hot_block_name`,hb.`_day` FROM `kpl_limit_up_record` hb WHERE hb.`_code`='{code}' AND {wheres} ORDER BY hb.`_day` DESC LIMIT 2) a GROUP BY a._day ORDER BY a._day DESC LIMIT 1"
mysqldb = mysql_data.Mysqldb()
return mysqldb.select_one(sql)
# 鑾峰彇浠ｇ爜鏈�杩戠殑鏉垮潡锛岃繑鍥瀃(鏉垮潡,鏃ユ湡)]
@classmethod
def get_latest_infos(cls, code, count, contains_today=True):
+ wheres = []
+ for b in constant.KPL_INVALID_BLOCKS:
+ wheres.append(f"hb.`_hot_block_name` != '{b}'")
+ wheres = " and ".join(wheres)
# 鍙幏鍙栨渶杩�180澶╃殑鏁版嵁
min_day = tool.date_sub(tool.get_now_date_str(), 180)
- sql = f"SELECT GROUP_CONCAT(_hot_block_name),`_day`,_blocks FROM (SELECT hb.`_hot_block_name`,hb.`_day`,hb._blocks FROM `kpl_limit_up_record` hb WHERE hb.`_code`='{code}' and hb.`_day` > '{min_day}' ORDER BY hb.`_day` DESC LIMIT 10) a GROUP BY a._day ORDER BY a._day DESC LIMIT {count}"
+ sql = f"SELECT GROUP_CONCAT(_hot_block_name),`_day`,_blocks FROM (SELECT hb.`_hot_block_name`,hb.`_day`,hb._blocks FROM `kpl_limit_up_record` hb WHERE hb.`_code`='{code}' and {wheres} and hb.`_day` > '{min_day}' ORDER BY hb.`_day` DESC LIMIT 10) a GROUP BY a._day ORDER BY a._day DESC LIMIT {count}"
mysqldb = mysql_data.Mysqldb()
results = mysqldb.select_all(sql)
if results and not contains_today and results[0][1] == tool.get_now_date_str():
@@ -176,21 +242,47 @@
@classmethod
def get_latest_blocks_set(cls, code):
- results = cls.get_latest_infos(code, 10, False)
+ results = cls.get_latest_infos(code, 2, False)
bs = set([b[0] for b in results])
return bs
+
+ @classmethod
+ def get_current_blocks(cls, code):
+ return cls.__current_code_reasons_dict.get(code)
+
+ @classmethod
+ def get_current_codes_by_block(cls, block):
+ return cls.__current_reason_codes_dict.get(block)
+
+ @classmethod
+ def get_current_reason_codes_dict(cls):
+ return copy.deepcopy(cls.__current_reason_codes_dict)
+
+ @classmethod
+ def get_current_limit_up_reason_codes_dict(cls):
+ return copy.deepcopy(cls.__current_limit_up_reason_codes_dict)
+
+ @classmethod
+ def get_current_reasons(cls):
+ if cls.__current_reason_codes_dict:
+ return cls.__current_reason_codes_dict.keys()
+ return set()
class KPLDataManager:
__latest_datas = {}
+ kpl_data_update_info = {}
+ __file_content_cache = {}
- def __save_in_file(self, key, datas):
+ @classmethod
+ def __save_in_file(cls, key, datas):
name = f"{tool.get_now_date_str()}_{key}.log"
path = f"{constant.CACHE_PATH}/{name}"
with open(path, 'w') as f:
f.write(json.dumps(datas))
- def __get_from_file(self, key, day=tool.get_now_date_str()):
+ @classmethod
+ def __get_from_file(cls, key, day=tool.get_now_date_str()):
name = f"{day}_{key}.log"
path = f"{constant.CACHE_PATH}/{name}"
if not os.path.exists(path):
@@ -201,7 +293,8 @@
return json.loads(lines[0])
return None
- def get_from_file(self, type, day):
+ @classmethod
+ def get_from_file(cls, type, day):
name = f"{day}_{type.value}.log"
path = f"{constant.CACHE_PATH}/{name}"
if not os.path.exists(path):
@@ -212,8 +305,19 @@
return json.loads(lines[0])
return None
+ @classmethod
+ def get_from_file_cache(cls, type, day):
+ key = f"{type}-{day}"
+ if key in cls.__file_content_cache:
+ return cls.__file_content_cache.get(key)
+ fdata = cls.get_from_file(type, day)
+ if fdata:
+ cls.__file_content_cache[key] = fdata
+ return fdata
+
+ @classmethod
# 鑾峰彇鏈�杩戝嚑澶╃殑鏁版嵁锛屾牴鎹棩鏈熷�掑簭杩斿洖
- def get_latest_from_file(self, type, count):
+ def get_latest_from_file(cls, type, count):
files = os.listdir(constant.CACHE_PATH)
file_name_list = []
for f in files:
@@ -234,17 +338,20 @@
return fresults
- def save_data(self, type, datas):
- self.__latest_datas[type] = datas
- self.__save_in_file(type, datas)
+ @classmethod
+ def save_data(cls, type, datas):
+ cls.kpl_data_update_info[type] = (tool.get_now_time_str(), len(datas))
+ cls.__latest_datas[type] = datas
+ cls.__save_in_file(type, datas)
- def get_data(self, type):
+ @classmethod
+ def get_data(cls, type):
type = type.value
- if type in self.__latest_datas:
- return self.__latest_datas[type]
- result = self.__get_from_file(type)
+ if type in cls.__latest_datas:
+ return cls.__latest_datas[type]
+ result = cls.__get_from_file(type)
if result is not None:
- self.__latest_datas[type] = result
+ cls.__latest_datas[type] = result
return result
@@ -279,7 +386,8 @@
if day in __limit_up_list_records_dict:
datas = __limit_up_list_records_dict[day]
else:
- datas = KPLDataManager().get_latest_from_file(KPLDataType.LIMIT_UP, 10)
+ logger_debug.info("浠庢枃浠朵腑鑾峰彇鍓嶅嚑澶╃殑瀹炴椂娑ㄥ仠鏁版嵁")
+ datas = KPLDataManager().get_latest_from_file(KPLDataType.LIMIT_UP, count + 2)
if datas:
# 淇濆瓨鏁版嵁
__limit_up_list_records_dict[day] = datas
@@ -292,8 +400,122 @@
return fresults
+def get_yesterday_limit_up_codes():
+ yesterday_limit_up_data_records = get_yesterday_current_limit_up_records()
+ yesterday_codes = set([x[0] for x in yesterday_limit_up_data_records])
+ return yesterday_codes
+
+
+def get_yesterday_current_limit_up_records():
+ yesterday_limit_up_data_records = get_current_limit_up_data_records(1)[0][1]
+ return yesterday_limit_up_data_records
+
+
+# 鑾峰彇鏈�杩戝嚑澶╂定鍋滃師鍥�
+__latest_current_limit_up_records = {}
+
+
+def get_latest_current_limit_up_records():
+ day = tool.get_now_date_str()
+ if day not in __latest_current_limit_up_records:
+ fdatas = get_current_limit_up_data_records(15)
+ __latest_current_limit_up_records[day] = fdatas
+ return __latest_current_limit_up_records.get(day)
+
+
+class PullTask:
+ # 鏈�杩戞洿鏂版椂闂�
+ __latest_update_time_dict = {}
+
+ @classmethod
+ def __upload_data(cls, type, datas):
+ root_data = {
+ "type": type,
+ "data": datas
+ }
+ requests.post("http://127.0.0.1:9004/upload_kpl_data", json.dumps(root_data))
+
+ @classmethod
+ def repaire_pull_task(cls):
+ """
+ 淇鎷夊彇浠诲姟
+ @return:
+ """
+ # 淇娑ㄥ仠
+ logger_debug.info("浠诲姟淇-寮�鐩樺暒锛氬惎鍔ㄤ慨澶�")
+ key = "limit_up"
+ if key not in cls.__latest_update_time_dict or time.time() - cls.__latest_update_time_dict[key] > 20:
+ logger_debug.info("浠诲姟淇-寮�鐩樺暒锛氭定鍋滃垪琛�")
+ # 澶т簬20s灏遍渶瑕佹洿鏂�
+ threading.Thread(target=cls.run_limit_up_task, daemon=True).start()
+
+ @classmethod
+ def run_limit_up_task(cls):
+ # 鍏抽棴log
+ log.close_print()
+ while True:
+ try:
+ if (tool.is_trade_time() and int(tool.get_now_time_str().replace(':', '')) > int("092530")):
+ results = kpl_api.getLimitUpInfoNew()
+ result = json.loads(results)
+ start_time = time.time()
+ cls.__upload_data("limit_up", result)
+ except Exception as e:
+ try:
+ logging.exception(e)
+ logger_debug.exception(e)
+ except:
+ pass
+ except:
+ pass
+ finally:
+ cls.__latest_update_time_dict["limit_up"] = time.time()
+ time.sleep(3)
+
+ @classmethod
+ # 杩愯鎷夊彇浠诲姟
+ def run_pull_task(cls):
+ def get_bidding_money():
+ # 绔炰环鏁版嵁涓婁紶
+ while True:
+ if int("092600") < int(tool.get_now_time_str().replace(":", "")) < int("092700"):
+ try:
+ results = kpl_api.daBanList(kpl_api.DABAN_TYPE_BIDDING)
+ result = json.loads(results)
+ cls.__upload_data("biddings", result)
+ except Exception as e:
+ pass
+ time.sleep(3)
+
+ def get_market_industry():
+ while True:
+ if tool.is_trade_time():
+ try:
+ results = kpl_api.getMarketIndustryRealRankingInfo()
+ result = json.loads(results)
+ cls.__upload_data("industry_rank", result)
+ except:
+ pass
+ time.sleep(3)
+
+ def get_market_jingxuan():
+ while True:
+ if tool.is_trade_time():
+ try:
+ results = kpl_api.getMarketJingXuanRealRankingInfo()
+ result = json.loads(results)
+ cls.__upload_data("jingxuan_rank", result)
+ except:
+ pass
+ time.sleep(3)
+
+ threading.Thread(target=cls.run_limit_up_task, daemon=True).start()
+ # threading.Thread(target=get_bidding_money, daemon=True).start()
+ # threading.Thread(target=get_market_industry, daemon=True).start()
+ # threading.Thread(target=get_market_jingxuan, daemon=True).start()
+
+
if __name__ == "__main__":
- fresults = get_current_limit_up_data_records(2)
- for d in fresults:
- print(d)
- get_current_limit_up_data_records(2)
+ print(get_latest_current_limit_up_records())
+ print(get_latest_current_limit_up_records())
+ input()
--
Gitblit v1.8.0