import json
|
import os
|
import threading
|
import time
|
|
import requests
|
|
import constant
|
from db.redis_manager import RedisUtils
|
from log_module.log import logger_kpl_limit_up_reason_change
|
from utils import tool
|
|
# 开盘啦历史涨停数据管理
|
from db import mysql_data as mysql_data, redis_manager as redis_manager
|
from third_data import kpl_util, kpl_api
|
from third_data.code_plate_key_manager import LimitUpCodesPlateKeyManager, CodesHisReasonAndBlocksManager
|
|
# 代码对应的涨停原因保存
|
from third_data.kpl_util import KPLPlatManager, KPLDataType
|
|
|
class KPLCodeLimitUpReasonManager:
|
__redisManager = redis_manager.RedisManager(3)
|
|
def __get_redis(self):
|
return self.__redisManager.getRedis()
|
|
def save_reason(self, code, reason):
|
RedisUtils.setex(self.__get_redis(), f"kpl_limitup_reason-{code}", tool.get_expire(), reason)
|
|
def list_all(self):
|
keys = RedisUtils.keys(self.__get_redis(), "kpl_limitup_reason-*")
|
dict_ = {}
|
for k in keys:
|
val = RedisUtils.get(self.__get_redis(), k)
|
dict_[k.split("-")[1]] = val
|
return dict_
|
|
|
class KPLLimitUpDataRecordManager:
|
total_datas = None
|
latest_datas = {}
|
latest_origin_datas = []
|
__kplPlatManager = KPLPlatManager()
|
__LimitUpCodesPlateKeyManager = LimitUpCodesPlateKeyManager()
|
__CodesPlateKeysManager = CodesHisReasonAndBlocksManager()
|
|
@classmethod
|
def __load_hist_and_blocks(cls, code):
|
# 有数据新增，加载历史原因与板块
|
his_reasons = cls.get_latest_infos(code, 10, False)
|
his_reasons = set([r[0] for r in his_reasons])
|
cls.__CodesPlateKeysManager.set_history_limit_up_reason(code, his_reasons)
|
try:
|
if not cls.__CodesPlateKeysManager.get_blocks(code):
|
results = kpl_api.getStockIDPlate(code)
|
bs = [r[1] for r in results]
|
cls.__CodesPlateKeysManager.set_blocks(code, bs)
|
except Exception as e:
|
pass
|
|
@classmethod
|
def save_record(cls, day, records):
|
# 统计代码所属板块
|
code_block_dict = {}
|
for data in records:
|
blocks = set(data[5].split("、"))
|
code = data[0]
|
for b in blocks:
|
if not code_block_dict.get(code):
|
code_block_dict[code] = set()
|
code_block_dict[code].add(b)
|
# 设置涨停数据
|
if records:
|
cls.latest_origin_datas = records
|
cls.__LimitUpCodesPlateKeyManager.set_today_limit_up([(r[0], r[5]) for r in records])
|
|
# 涨停数据记录
|
mysqldb = mysql_data.Mysqldb()
|
# 统计涨停原因和概念代码
|
plats = {}
|
for d in records:
|
plats[d[5]] = d[9]
|
for p in plats:
|
cls.__kplPlatManager.save_plat(plats[p], p)
|
|
for d in records:
|
# (代码, 名称, 首次涨停时间, 最近涨停时间, 几板, 涨停原因, 板块, 实际流通, 主力净额,涨停原因代码,涨停原因代码数量)
|
code = d[0]
|
_id = f"{day}_{code}_{d[5]}"
|
|
result = mysqldb.select_one("select * from kpl_limit_up_record where _id='{}'".format(_id))
|
if not result:
|
mysqldb.execute(
|
f"insert into kpl_limit_up_record(_id,_day,_hot_block_name,_code,_code_name,_limit_up_time,_blocks,_latest_limit_up_time,_update_time,_create_time,_hot_block_code_count,_limit_up_high_info,_zylt_val) values('{_id}','{day}','{d[5]}','{d[0]}','{d[1]}','{d[2]}','{d[6]}','{d[3]}',now(),now(),{d[10]},'{d[4]}',{d[7]})")
|
cls.__load_hist_and_blocks(code)
|
else:
|
if _id in cls.latest_datas and json.dumps(cls.latest_datas.get(_id)) != json.dumps(d):
|
mysqldb.execute(
|
f"update kpl_limit_up_record set _latest_limit_up_time='{d[3]}',_limit_up_time='{d[2]}',_hot_block_code_count={d[10]},_limit_up_high_info='{d[4]}' ,_update_time=now() where _id='{_id}'")
|
cls.latest_datas[_id] = d
|
cls.latest_datas[_id] = d
|
|
# 获取原来的代码所属板块,删除之前错误的板块
|
old_datas = KPLLimitUpDataRecordManager.list_by_code(code, day)
|
if old_datas:
|
for dd in old_datas:
|
if dd[2] not in code_block_dict[code]:
|
mysqldb.execute(f"delete from kpl_limit_up_record where _id='{dd[0]}'")
|
logger_kpl_limit_up_reason_change.info(f"code-{dd[3]}:{dd[2]}-{code_block_dict[code]}")
|
# 板块更改过
|
mysqldb.execute(
|
f"update kpl_limit_up_record set _hot_block_change = f'{dd[2]}' where _day='{dd[1]}' and _code='{code}'")
|
|
cls.__LimitUpCodesPlateKeyManager.set_today_limit_up_reason_change(code, dd[2],
|
code_block_dict[code])
|
|
if dd[0] in cls.latest_datas:
|
cls.latest_datas.pop(dd[0])
|
cls.total_datas = KPLLimitUpDataRecordManager.list_all(tool.get_now_date_str())
|
|
@classmethod
|
def load_total_datas(cls):
|
cls.total_datas = KPLLimitUpDataRecordManager.list_all(tool.get_now_date_str())
|
cls.__LimitUpCodesPlateKeyManager.set_today_total_limit_up([(r[3], r[2]) for r in cls.total_datas])
|
for d in cls.total_datas:
|
cls.__load_hist_and_blocks(d[3])
|
|
@staticmethod
|
def list_all(day):
|
mysqldb = mysql_data.Mysqldb()
|
return mysqldb.select_all(f"select * from kpl_limit_up_record where _day='{day}'")
|
|
@staticmethod
|
def list_by_code(code, day):
|
mysqldb = mysql_data.Mysqldb()
|
return mysqldb.select_all(f"select * from kpl_limit_up_record where _code='{code}' and _day='{day}'")
|
|
@staticmethod
|
def list_by_block(block_name, day):
|
mysqldb = mysql_data.Mysqldb()
|
return mysqldb.select_all(
|
f"select * from kpl_limit_up_record where _hot_block_name='{block_name}' and _day='{day}'")
|
|
@staticmethod
|
def list_blocks_with_day(days):
|
mysqldb = mysql_data.Mysqldb()
|
sql = "select _hot_block_name,_day from kpl_limit_up_record where "
|
wheres = []
|
for day in days:
|
wheres.append(f"_day = '{day}'")
|
sql += " or ".join(wheres)
|
sql += " group by _hot_block_name,_day"
|
|
results = mysqldb.select_all(sql)
|
return results
|
|
@staticmethod
|
def get_latest_blocks(code):
|
wheres = []
|
for b in constant.KPL_INVALID_BLOCKS:
|
wheres.append(f"hb.`_hot_block_name` != '{b}'")
|
wheres = " and ".join(wheres)
|
sql = f"SELECT GROUP_CONCAT(_hot_block_name) FROM (SELECT hb.`_hot_block_name`,hb.`_day` FROM `kpl_limit_up_record` hb WHERE hb.`_code`='{code}' AND {wheres} ORDER BY hb.`_day` DESC LIMIT 2) a GROUP BY a._day ORDER BY a._day DESC LIMIT 1"
|
mysqldb = mysql_data.Mysqldb()
|
return mysqldb.select_one(sql)
|
|
# 获取代码最近的板块，返回[(板块,日期)]
|
@classmethod
|
def get_latest_infos(cls, code, count, contains_today=True):
|
wheres = []
|
for b in constant.KPL_INVALID_BLOCKS:
|
wheres.append(f"hb.`_hot_block_name` != '{b}'")
|
wheres = " and ".join(wheres)
|
# 只获取最近180天的数据
|
min_day = tool.date_sub(tool.get_now_date_str(), 180)
|
sql = f"SELECT GROUP_CONCAT(_hot_block_name),`_day`,_blocks FROM (SELECT hb.`_hot_block_name`,hb.`_day`,hb._blocks FROM `kpl_limit_up_record` hb WHERE hb.`_code`='{code}' and {wheres} and hb.`_day` > '{min_day}' ORDER BY hb.`_day` DESC LIMIT 10) a GROUP BY a._day ORDER BY a._day DESC LIMIT {count}"
|
mysqldb = mysql_data.Mysqldb()
|
results = mysqldb.select_all(sql)
|
if results and not contains_today and results[0][1] == tool.get_now_date_str():
|
return results[1:]
|
return results
|
|
@classmethod
|
def get_latest_blocks_set(cls, code):
|
results = cls.get_latest_infos(code, 2, False)
|
bs = set([b[0] for b in results])
|
return bs
|
|
|
class KPLDataManager:
|
__latest_datas = {}
|
kpl_data_update_info = {}
|
|
@classmethod
|
def __save_in_file(cls, key, datas):
|
name = f"{tool.get_now_date_str()}_{key}.log"
|
path = f"{constant.CACHE_PATH}/{name}"
|
with open(path, 'w') as f:
|
f.write(json.dumps(datas))
|
|
@classmethod
|
def __get_from_file(cls, key, day=tool.get_now_date_str()):
|
name = f"{day}_{key}.log"
|
path = f"{constant.CACHE_PATH}/{name}"
|
if not os.path.exists(path):
|
return None
|
with open(path, 'r') as f:
|
lines = f.readlines()
|
if lines:
|
return json.loads(lines[0])
|
return None
|
|
@classmethod
|
def get_from_file(cls, type, day):
|
name = f"{day}_{type.value}.log"
|
path = f"{constant.CACHE_PATH}/{name}"
|
if not os.path.exists(path):
|
return None
|
with open(path, 'r') as f:
|
lines = f.readlines()
|
if lines:
|
return json.loads(lines[0])
|
return None
|
|
@classmethod
|
# 获取最近几天的数据，根据日期倒序返回
|
def get_latest_from_file(cls, type, count):
|
files = os.listdir(constant.CACHE_PATH)
|
file_name_list = []
|
for f in files:
|
if f[10:] == f"_{type.value}.log":
|
file_name_list.append((f.split("_")[0], f))
|
|
file_name_list.sort(key=lambda x: x[0], reverse=True)
|
file_name_list = file_name_list[:count]
|
fresults = []
|
for file in file_name_list:
|
path = f"{constant.CACHE_PATH}/{file[1]}"
|
if not os.path.exists(path):
|
continue
|
with open(path, 'r') as f:
|
lines = f.readlines()
|
if lines:
|
fresults.append((file[0], json.loads(lines[0])))
|
|
return fresults
|
|
@classmethod
|
def save_data(cls, type, datas):
|
cls.kpl_data_update_info[type] = (tool.get_now_time_str(), len(datas))
|
cls.__latest_datas[type] = datas
|
cls.__save_in_file(type, datas)
|
|
@classmethod
|
def get_data(cls, type):
|
type = type.value
|
if type in cls.__latest_datas:
|
return cls.__latest_datas[type]
|
result = cls.__get_from_file(type)
|
if result is not None:
|
cls.__latest_datas[type] = result
|
return result
|
|
|
def load_history_limit_up():
|
for file_name in os.listdir(f"{constant.get_path_prefix()}/kpl/his"):
|
if file_name.find("HisDaBanList_1.log") < 0:
|
continue
|
day = file_name[:10]
|
with open(f"{constant.get_path_prefix()}/kpl/his/{file_name}", 'r', encoding="utf-16") as f:
|
lines = f.readlines()
|
line = lines[0]
|
result = json.loads(line)
|
list_ = kpl_util.parseDaBanData(result, kpl_util.DABAN_TYPE_LIMIT_UP)
|
# KPLLimitUpDataRecordManager.save_record(day, list_)
|
for r in list_:
|
print(r[-1], r[5])
|
KPLPlatManager().save_plat(r[-1], r[5])
|
|
# print(day, list_)
|
|
|
# 历史涨停列表
|
__limit_up_list_records_dict = {}
|
|
|
# 获取最近几天的实时涨停信息
|
# 返回格式（[日期,数据]）
|
def get_current_limit_up_data_records(count):
|
fresults = []
|
day = tool.get_now_date_str()
|
datas = []
|
if day in __limit_up_list_records_dict:
|
datas = __limit_up_list_records_dict[day]
|
else:
|
datas = KPLDataManager().get_latest_from_file(KPLDataType.LIMIT_UP, 10)
|
if datas:
|
# 保存数据
|
__limit_up_list_records_dict[day] = datas
|
for i in range(len(datas)):
|
if datas[i][0] == day:
|
continue
|
fresults.append(datas[i])
|
if len(fresults) >= count:
|
break
|
return fresults
|
|
|
def get_yesterday_limit_up_codes():
|
yesterday_limit_up_data_records = get_current_limit_up_data_records(1)[0][1]
|
yesterday_codes = set([x[0] for x in yesterday_limit_up_data_records])
|
return yesterday_codes
|
|
|
# 运行拉取任务
|
def run_pull_task():
|
def __upload_data(type, datas):
|
root_data = {
|
"type": type,
|
"data": datas
|
}
|
requests.post("http://127.0.0.1:9004/upload_kpl_data", json.dumps(root_data))
|
|
def get_limit_up():
|
while True:
|
if tool.is_trade_time():
|
try:
|
results = kpl_api.getLimitUpInfo()
|
result = json.loads(results)
|
__upload_data("limit_up", result)
|
except Exception as e:
|
pass
|
time.sleep(3)
|
|
def get_bidding_money():
|
# 竞价数据上传
|
while True:
|
if int("092600") < int(tool.get_now_time_str().replace(":", "")) < int("092700"):
|
try:
|
results = kpl_api.daBanList(kpl_api.DABAN_TYPE_BIDDING)
|
result = json.loads(results)
|
__upload_data("biddings", result)
|
except Exception as e:
|
pass
|
time.sleep(3)
|
|
def get_market_industry():
|
while True:
|
if tool.is_trade_time():
|
try:
|
results = kpl_api.getMarketIndustryRealRankingInfo()
|
result = json.loads(results)
|
__upload_data("industry_rank", result)
|
except:
|
pass
|
time.sleep(3)
|
|
def get_market_jingxuan():
|
while True:
|
if tool.is_trade_time():
|
try:
|
results = kpl_api.getMarketJingXuanRealRankingInfo()
|
result = json.loads(results)
|
__upload_data("jingxuan_rank", result)
|
except:
|
pass
|
time.sleep(3)
|
|
threading.Thread(target=get_limit_up, daemon=True).start()
|
# threading.Thread(target=get_bidding_money, daemon=True).start()
|
threading.Thread(target=get_market_industry, daemon=True).start()
|
threading.Thread(target=get_market_jingxuan, daemon=True).start()
|
|
|
if __name__ == "__main__":
|
# run_pull_task()
|
# input()
|
results = kpl_api.daBanList(kpl_api.DABAN_TYPE_LIMIT_UP)
|
results = json.loads(results)
|
results = results["list"]
|
for result in results:
|
print(result)
|
