import copy import json import logging import os import threading import time import requests from db.redis_manager_delegate import RedisUtils from log_module import async_log_util, log from utils import tool, constant # 开盘啦历史涨停数据管理 from db import mysql_data_delegate as mysql_data, redis_manager_delegate as redis_manager from log_module.log import logger_kpl_limit_up_reason_change, logger_debug, \ logger_kpl_open_limit_up, logger_kpl_block_can_buy from third_data import kpl_util, kpl_api, kpl_block_util # 代码对应的涨停原因保存 from third_data.kpl_util import KPLPlatManager, KPLDataType class KPLLimitUpDataRecordManager: total_datas = None latest_datas = {} latest_origin_datas = [] __kplPlatManager = KPLPlatManager() __current_code_reasons_dict = {} # 当前涨停原因+推荐原因的代码集合 __current_reason_codes_dict = {} # 当前涨停原因的代码集合 __current_limit_up_reason_codes_dict = {} __records_cache = {} record_code_dict = {} @classmethod def save_record(cls, day, records): # 统计炸板 try: last_codes = set() if cls.latest_origin_datas: last_codes = set([x[0] for x in cls.latest_origin_datas]) now_codes = set() if records: now_codes = set([x[0] for x in records]) open_limit_up_codes = last_codes - now_codes if open_limit_up_codes: logger_kpl_open_limit_up.info(f"炸板代码：{open_limit_up_codes}") except Exception as e: pass # 统计代码所属板块 code_block_dict = {} for data in records: cls.record_code_dict[data[0]] = data blocks = set(data[5].split("、")) code = data[0] for b in blocks: if not code_block_dict.get(code): code_block_dict[code] = set() code_block_dict[code].add(b) # 设置涨停数据 if records: cls.latest_origin_datas = records code_reasons_dict = {} reason_codes_dict = {} limit_up_reason_codes_dict = {} for d in records: if d[5] not in limit_up_reason_codes_dict: limit_up_reason_codes_dict[d[5]] = set() limit_up_reason_codes_dict[d[5]].add(d[0]) # 涨停原因 + 推荐原因 bs = {d[5]} if d[6]: bs |= set(d[6].split("、")) code_reasons_dict[d[0]] = bs for b in bs: if b not in reason_codes_dict: reason_codes_dict[b] = set() reason_codes_dict[b].add(d[0]) cls.__current_code_reasons_dict = code_reasons_dict cls.__current_reason_codes_dict = reason_codes_dict cls.__current_limit_up_reason_codes_dict = limit_up_reason_codes_dict # 涨停数据记录 mysqldb = mysql_data.Mysqldb() # 统计涨停原因和概念代码 plats = {} for d in records: plats[d[5]] = d[9] for p in plats: cls.__kplPlatManager.save_plat(plats[p], p) for d in records: # (代码, 名称, 首次涨停时间, 最近涨停时间, 几板, 涨停原因, 板块, 实际流通, 主力净额,涨停原因代码,涨停原因代码数量) code = d[0] _id = f"{day}_{code}_{d[5]}" result = mysqldb.select_one("select * from kpl_limit_up_record where _id='{}'".format(_id)) if not result: mysqldb.execute( f"insert into kpl_limit_up_record(_id,_day,_hot_block_name,_code,_code_name,_limit_up_time,_blocks,_latest_limit_up_time,_update_time,_create_time,_hot_block_code_count,_limit_up_high_info,_zylt_val) values('{_id}','{day}','{d[5]}','{d[0]}','{d[1]}','{d[2]}','{d[6]}','{d[3]}',now(),now(),{d[10]},'{d[4]}',{d[7]})") else: if _id in cls.latest_datas and json.dumps(cls.latest_datas.get(_id)) != json.dumps(d): mysqldb.execute( f"update kpl_limit_up_record set _latest_limit_up_time='{d[3]}',_limit_up_time='{d[2]}',_hot_block_code_count={d[10]},_limit_up_high_info='{d[4]}' ,_update_time=now() where _id='{_id}'") cls.latest_datas[_id] = d cls.latest_datas[_id] = d # 获取原来的代码所属板块,删除之前错误的板块 old_datas = KPLLimitUpDataRecordManager.list_by_code(code, day) if old_datas: for dd in old_datas: if dd[2] not in code_block_dict[code]: mysqldb.execute(f"delete from kpl_limit_up_record where _id='{dd[0]}'") logger_kpl_limit_up_reason_change.info(f"code-{dd[3]}:{dd[2]}-{code_block_dict[code]}") # 板块更改过 mysqldb.execute( f"update kpl_limit_up_record set _hot_block_change = f'{dd[2]}' where _day='{dd[1]}' and _code='{code}'") if dd[0] in cls.latest_datas: cls.latest_datas.pop(dd[0]) cls.total_datas = KPLLimitUpDataRecordManager.list_all(tool.get_now_date_str()) @classmethod def load_total_datas(cls): cls.total_datas = KPLLimitUpDataRecordManager.list_all(tool.get_now_date_str()) @staticmethod def list_all(day): mysqldb = mysql_data.Mysqldb() return mysqldb.select_all(f"select * from kpl_limit_up_record where _day='{day}'") @classmethod def list_all_cache(cls, day): if day in cls.__records_cache: return cls.__records_cache[day] fdata = cls.list_all(day) if fdata: cls.__records_cache[day] = fdata return fdata @staticmethod def list_by_code(code, day): mysqldb = mysql_data.Mysqldb() return mysqldb.select_all(f"select * from kpl_limit_up_record where _code='{code}' and _day='{day}'") @staticmethod def list_by_block(block_name, day): mysqldb = mysql_data.Mysqldb() return mysqldb.select_all( f"select * from kpl_limit_up_record where _hot_block_name='{block_name}' and _day='{day}'") @staticmethod def list_blocks_with_day(days): mysqldb = mysql_data.Mysqldb() sql = "select _hot_block_name,_day from kpl_limit_up_record where " wheres = [] for day in days: wheres.append(f"_day = '{day}'") sql += " or ".join(wheres) sql += " group by _hot_block_name,_day" results = mysqldb.select_all(sql) return results @classmethod def get_current_blocks(cls, code): return cls.__current_code_reasons_dict.get(code) @classmethod def get_current_codes_by_block(cls, block): return cls.__current_reason_codes_dict.get(block) @classmethod def get_current_reason_codes_dict(cls): return copy.deepcopy(cls.__current_reason_codes_dict) @classmethod def get_current_limit_up_reason_codes_dict(cls): return copy.deepcopy(cls.__current_limit_up_reason_codes_dict) @classmethod def get_current_reasons(cls): if cls.__current_reason_codes_dict: return cls.__current_reason_codes_dict.keys() return set() # 代码精选板块管理 class KPLCodeJXBlockManager: __db = constant.REDIS_DB __redisManager = redis_manager.RedisManager(constant.REDIS_DB) __code_blocks = {} # 备用 __code_by_blocks = {} __instance = None def __new__(cls, *args, **kwargs): if not cls.__instance: cls.__instance = super(KPLCodeJXBlockManager, cls).__new__(cls, *args, **kwargs) return cls.__instance def __get_redis(self): return self.__redisManager.getRedis() def save_jx_blocks(self, code, blocks: list, current_limit_up_blocks: set, by=False): if not blocks: return final_blocks = copy.deepcopy(blocks) if len(blocks) > 2: final_blocks.clear() for b in blocks: if b not in constant.KPL_INVALID_BLOCKS: final_blocks.append(b) if len(final_blocks) < 2: final_blocks = blocks # 保存前2条数据 if by: RedisUtils.setex_async(self.__db, f"kpl_jx_blocks_by-{code}", tool.get_expire(), json.dumps(final_blocks)) self.__code_by_blocks[code] = (final_blocks, time.time()) else: RedisUtils.setex_async(self.__db, f"kpl_jx_blocks-{code}", tool.get_expire(), json.dumps(final_blocks)) self.__code_blocks[code] = (final_blocks, time.time()) # 获取精选板块 def get_jx_blocks(self, code, by=False): if by: if code in self.__code_by_blocks: return self.__code_by_blocks[code] val = RedisUtils.get(self.__get_redis(), f"kpl_jx_blocks_by-{code}") if val is None: return None else: val = json.loads(val) self.__code_by_blocks[code] = val return self.__code_by_blocks[code] else: if code in self.__code_blocks: return self.__code_blocks[code] val = RedisUtils.get(self.__get_redis(), f"kpl_jx_blocks-{code}") if val is None: return None else: val = json.loads(val) self.__code_blocks[code] = val return self.__code_blocks[code] def get_jx_blocks_cache(self, code, by=False): if by: return self.__code_by_blocks.get(code) else: return self.__code_blocks.get(code) # 从网络上加载精选板块, 当前涨停的板块 def load_jx_blocks(self, code, buy_1_price, limit_up_price, current_limit_up_blocks): try: # logger_kpl_block_can_buy.info(f"准备更新精选板块：{code}-{buy_1_price}-{limit_up_price}") if limit_up_price and buy_1_price: # 处理买1,卖1信息 pre_close_price = round(float(limit_up_price) / tool.get_limit_up_rate(code), 2) # 如果涨幅大于5%就读取板块 price_rate = (buy_1_price - pre_close_price) / pre_close_price if price_rate > 0.05 * (tool.get_limit_up_rate(code) - 1) * 10: jx_blocks_info = self.get_jx_blocks_cache(code) if not jx_blocks_info: start_time = time.time() blocks = kpl_api.getCodeBlocks(code) async_log_util.info(logger_kpl_block_can_buy, f"{code}:获取到精选板块-{blocks} 耗时:{int(time.time() - start_time)}s") self.save_jx_blocks(code, blocks, current_limit_up_blocks) else: # 还没涨停的需要更新精选板块 更新精选板块 if abs(float(buy_1_price) - float(limit_up_price)) >= 0.001: # 非涨停状态 UPDATE_TIME_SPACE = 5 * 60 time_diff = tool.trade_time_sub(tool.get_now_time_str(), "09:30:00") if time_diff < 0: UPDATE_TIME_SPACE = 60 * 60 else: UPDATE_TIME_SPACE = int(time_diff / 30) + 60 if UPDATE_TIME_SPACE > 5 * 60: UPDATE_TIME_SPACE = 5 * 60 if time.time() - jx_blocks_info[1] > UPDATE_TIME_SPACE: start_time = time.time() # 距离上次更新时间过去了5分钟 blocks = kpl_api.getCodeBlocks(code) async_log_util.info(logger_kpl_block_can_buy, f"{code}:获取到精选板块（更新）-{blocks} 耗时:{int(time.time() - start_time)}s") self.save_jx_blocks(code, blocks, current_limit_up_blocks) elif price_rate > 0.03 * (tool.get_limit_up_rate(code) - 1) * 10: # 添加备用板块 if not self.get_jx_blocks_cache(code, by=True): start_time = time.time() blocks = kpl_api.getCodeBlocks(code) self.save_jx_blocks(code, blocks, current_limit_up_blocks, by=True) async_log_util.info(logger_kpl_block_can_buy, f"{code}:获取到精选板块(备用)-{blocks} 耗时:{int(time.time() - start_time)}s") except Exception as e: logger_kpl_block_can_buy.error(f"{code} 获取板块出错") logger_kpl_block_can_buy.exception(e) class KPLDataManager: __latest_datas = {} kpl_data_update_info = {} __file_content_cache = {} @classmethod def __save_in_file(cls, key, datas): name = f"{tool.get_now_date_str()}_{key}.log" path = f"{constant.CACHE_PATH}/{name}" with open(path, 'w') as f: f.write(json.dumps(datas)) @classmethod def __get_from_file(cls, key, day=tool.get_now_date_str()): name = f"{day}_{key}.log" path = f"{constant.CACHE_PATH}/{name}" if not os.path.exists(path): return None with open(path, 'r') as f: lines = f.readlines() if lines: return json.loads(lines[0]) return None @classmethod def get_from_file(cls, type, day): name = f"{day}_{type.value}.log" path = f"{constant.CACHE_PATH}/{name}" if not os.path.exists(path): return None with open(path, 'r') as f: lines = f.readlines() if lines: return json.loads(lines[0]) return None @classmethod def get_from_file_cache(cls, type, day): key = f"{type}-{day}" if key in cls.__file_content_cache: return cls.__file_content_cache.get(key) fdata = cls.get_from_file(type, day) if fdata: cls.__file_content_cache[key] = fdata return fdata @classmethod # 获取最近几天的数据，根据日期倒序返回 def get_latest_from_file(cls, type, count): files = os.listdir(constant.CACHE_PATH) file_name_list = [] for f in files: if f[10:] == f"_{type.value}.log": file_name_list.append((f.split("_")[0], f)) file_name_list.sort(key=lambda x: x[0], reverse=True) file_name_list = file_name_list[:count] fresults = [] for file in file_name_list: path = f"{constant.CACHE_PATH}/{file[1]}" if not os.path.exists(path): continue with open(path, 'r') as f: lines = f.readlines() if lines: fresults.append((file[0], json.loads(lines[0]))) return fresults @classmethod def save_data(cls, type, datas): cls.kpl_data_update_info[type] = (tool.get_now_time_str(), len(datas)) cls.__latest_datas[type] = datas cls.__save_in_file(type, datas) @classmethod def get_data(cls, type): type = type.value if type in cls.__latest_datas: return cls.__latest_datas[type] result = cls.__get_from_file(type) if result is not None: cls.__latest_datas[type] = result return result class CodePlateKeyBuyManager: # 无板块 BLOCK_TYPE_NONE = -1 # 一般板块 BLOCK_TYPE_COMMON = 0 # 强势板块 BLOCK_TYPE_STRONG = 1 # 猛拉板块 BLOCK_TYPE_SOON_LIMIT_UP = 2 # 潜伏板块 BLOCK_TYPE_START_UP = 3 __can_buy_compute_result_dict = {} @classmethod def __is_block_can_buy_new(cls, code, block, current_limit_up_datas, code_limit_up_reasons_dict, current_limit_up_block_codes_dict, yesterday_current_limit_up_codes): """ 该票的板块是否可以买 @param code: @param block: @param current_limit_up_datas: 当前涨停数据 @param code_limit_up_reasons_dict: 代码的涨停原因 @param current_limit_up_block_codes_dict: 今日涨停板块对应的代码 @param yesterday_current_limit_up_codes: 昨日涨停代码 @return: 返回(前排开1代码集合,高位板集合,创业板集合,身位) """ block_codes = current_limit_up_block_codes_dict.get(block) if block_codes is None: block_codes = set() # 判断开1数量 current_open_limit_up_codes = kpl_block_util.get_shsz_open_limit_up_codes_current(code, block, current_limit_up_datas) # 创业板数量 gem_codes = set() for c in block_codes: if c.find("30") == 0: gem_codes.add(code) # 高位板数量 high_codes = set() for c in block_codes: if c in yesterday_current_limit_up_codes: high_codes.add(c) # 获取主板实时身位,剔除高位板 exclude_first_codes = set() current_shsz_rank, front_current_shsz_rank_codes = kpl_block_util.get_code_current_rank(code, block, current_limit_up_datas, code_limit_up_reasons_dict, yesterday_current_limit_up_codes, exclude_first_codes, len( current_open_limit_up_codes), shsz=True) return current_open_limit_up_codes, high_codes, gem_codes, current_shsz_rank @classmethod def get_plate_keys(cls, code): """ 获取代码的板块 :param code: :return: 返回代码板块 """ blocks_info = KPLCodeJXBlockManager().get_jx_blocks_cache(code) if not blocks_info: blocks_info = KPLCodeJXBlockManager().get_jx_blocks_cache(code, by=True) if blocks_info: return blocks_info[0] return None # 获取可以买的板块 # current_limit_up_datas: 今日实时涨停 # latest_2_day_limit_up_datas：最近2天的实时涨停（不含今日） # limit_up_record_datas：今日历史涨停 # yesterday_current_limit_up_codes ： 昨日涨停代码 # before_blocks_dict：历史涨停原因 # 返回板块的计算结果[(板块名称,是否可买,是否是独苗,信息)] @classmethod def get_can_buy_block(cls, code, current_limit_up_datas, yesterday_current_limit_up_codes): # 加载涨停代码的目标板块 def load_limit_up_codes_block(): if current_limit_up_datas: for d in current_limit_up_datas: blocks = {d[5]} if d[6]: blocks |= set(d[6].split('、')) blocks = blocks - constant.KPL_INVALID_BLOCKS code_limit_up_reasons_dict[d[0]] = blocks return code_limit_up_reasons_dict if current_limit_up_datas is None: current_limit_up_datas = [] # 获取目标代码板块 keys = cls.get_plate_keys(code) # log.logger_kpl_debug.info("{}最终关键词：{}", code, keys) # 涨停列表中匹配关键词，返回（板块:代码集合），代码集合中已经排除自身 fresults = [] if not keys: return fresults, set() code_limit_up_reasons_dict = {} current_limit_up_block_codes_dict = {} load_limit_up_codes_block() for c in code_limit_up_reasons_dict: bs = code_limit_up_reasons_dict[c] for b in bs: if b not in current_limit_up_block_codes_dict: current_limit_up_block_codes_dict[b] = set() current_limit_up_block_codes_dict[b].add(c) for block in keys: # 前排开1数量, 高位板数量, 创业板数量, 身位 open_limit_up_codes, high_codes, gem_codes, rank = cls.__is_block_can_buy_new( code, block, current_limit_up_datas, code_limit_up_reasons_dict, current_limit_up_block_codes_dict, yesterday_current_limit_up_codes) fresults.append((block, open_limit_up_codes, high_codes, gem_codes, rank)) return fresults, keys # 返回:(可以买的板块列表, 是否是独苗, 消息简介,可买的强势主线, 激进买入板块列表) @classmethod def statistic_block_infos(cls, code, current_limit_up_datas): """ 统计板块信息 :param code: :param current_limit_up_datas: :return: [(板块, 前排开1数量, 高位板数量, 创业板数量, 身位)] """ yesterday_limit_up_codes = get_yesterday_limit_up_codes() blocks_compute_results, keys = cls.get_can_buy_block(code, current_limit_up_datas, yesterday_limit_up_codes) return blocks_compute_results def load_history_limit_up(): for file_name in os.listdir(f"{constant.get_path_prefix()}/kpl/his"): if file_name.find("HisDaBanList_1.log") < 0: continue day = file_name[:10] with open(f"{constant.get_path_prefix()}/kpl/his/{file_name}", 'r', encoding="utf-16") as f: lines = f.readlines() line = lines[0] result = json.loads(line) list_ = kpl_util.parseDaBanData(result, kpl_util.DABAN_TYPE_LIMIT_UP) # KPLLimitUpDataRecordManager.save_record(day, list_) for r in list_: print(r[-1], r[5]) KPLPlatManager().save_plat(r[-1], r[5]) # print(day, list_) # 历史涨停列表 __limit_up_list_records_dict = {} # 获取最近几天的实时涨停信息 # 返回格式（[日期,数据]） def get_current_limit_up_data_records(count): fresults = [] day = tool.get_now_date_str() datas = [] if day in __limit_up_list_records_dict: datas = __limit_up_list_records_dict[day] else: logger_debug.info("从文件中获取前几天的实时涨停数据") datas = KPLDataManager().get_latest_from_file(KPLDataType.LIMIT_UP, count + 2) if datas: # 保存数据 __limit_up_list_records_dict[day] = datas for i in range(len(datas)): if datas[i][0] == day: continue fresults.append(datas[i]) if len(fresults) >= count: break return fresults def get_yesterday_limit_up_codes(): yesterday_limit_up_data_records = get_yesterday_current_limit_up_records() yesterday_codes = set([x[0] for x in yesterday_limit_up_data_records]) return yesterday_codes def get_yesterday_current_limit_up_records(): yesterday_limit_up_data_records = get_current_limit_up_data_records(1)[0][1] return yesterday_limit_up_data_records # 获取最近几天涨停原因 __latest_current_limit_up_records = {} def get_latest_current_limit_up_records(): day = tool.get_now_date_str() if day not in __latest_current_limit_up_records: fdatas = get_current_limit_up_data_records(15) __latest_current_limit_up_records[day] = fdatas return __latest_current_limit_up_records.get(day) class PullTask: # 最近更新时间 __latest_update_time_dict = {} @classmethod def __upload_data(cls, type, datas): root_data = { "type": type, "data": datas } requests.post("http://127.0.0.1:9004/upload_kpl_data", json.dumps(root_data)) @classmethod def repaire_pull_task(cls): """ 修复拉取任务 @return: """ # 修复涨停 logger_debug.info("任务修复-开盘啦：启动修复") key = "limit_up" if key not in cls.__latest_update_time_dict or time.time() - cls.__latest_update_time_dict[key] > 20: logger_debug.info("任务修复-开盘啦：涨停列表") # 大于20s就需要更新 threading.Thread(target=cls.run_limit_up_task, daemon=True).start() @classmethod def run_limit_up_task(cls, limit_up_result_callback): # 关闭log log.close_print() while True: try: if (tool.is_trade_time() and int(tool.get_now_time_str().replace(':', '')) > int("092530")): results = kpl_api.getLimitUpInfoNew() result = json.loads(results) start_time = time.time() limit_up_result_callback(result) except Exception as e: try: logging.exception(e) logger_debug.exception(e) except: pass except: pass finally: cls.__latest_update_time_dict["limit_up"] = time.time() time.sleep(3) @classmethod # 运行拉取任务 def run_pull_task(cls): threading.Thread(target=cls.run_limit_up_task, daemon=True).start() # threading.Thread(target=get_bidding_money, daemon=True).start() # threading.Thread(target=get_market_industry, daemon=True).start() # threading.Thread(target=get_market_jingxuan, daemon=True).start() if __name__ == "__main__": print(get_latest_current_limit_up_records()) print(get_latest_current_limit_up_records()) input()