src/main/java/com/yeshi/buwan/dao/ResourceVideoDao.java
@@ -14,6 +14,7 @@ /** * 根据视频ID和来源ID查询 * * @param videoId * @param resourceId * @return @@ -26,4 +27,10 @@ return null; } public long countByVideoIdAndResourceId(String videoId, String resourceId) { return getCount("from ResourceVideo rv where rv.video.id=? and rv.resource.id=?", new Serializable[]{videoId, resourceId}); } } src/main/java/com/yeshi/buwan/dao/VideoInfoDao.java
@@ -1,6 +1,9 @@ package com.yeshi.buwan.dao; import com.yeshi.buwan.dao.base.BaseDao; import org.hibernate.HibernateException; import org.hibernate.Session; import org.springframework.orm.hibernate4.HibernateCallback; import org.springframework.stereotype.Repository; import com.yeshi.buwan.domain.VideoInfo; @@ -45,4 +48,21 @@ return list(hql, videos); } /** * 统计视频附加信息(包含地区,分类,来源) * * @param videoId -视频ID */ public void statisticVideoExtraInfo(final String videoId) { excute(new HibernateCallback() { @Override public Object doInHibernate(Session session) throws HibernateException { session.getTransaction().begin(); session.createSQLQuery(String.format("call copyvideotemp(%s)", videoId)).executeUpdate(); session.getTransaction().commit(); return null; } }); } } src/main/java/com/yeshi/buwan/dao/juhe/funtv/FunTVAlbumDao.java
@@ -1,10 +1,12 @@ package com.yeshi.buwan.dao.juhe.funtv; import com.yeshi.buwan.dao.base.MongodbBaseDao; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Repository; import com.yeshi.buwan.funtv.entity.FunTVAlbum; @Repository public class FunTVAlbumDao extends MongodbBaseDao<FunTVAlbum> { } src/main/java/com/yeshi/buwan/dao/juhe/funtv/VideoFunTV2Dao.java
@@ -7,6 +7,8 @@ import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Repository; import java.util.List; @Repository public class VideoFunTV2Dao extends MongodbBaseDao<VideoFunTV2> { @@ -36,4 +38,18 @@ return findOne(query); } /** * 查询所有 * * @param start * @param count * @return */ public List<VideoFunTV2> listAll(int start, int count) { Query query = new Query(); query.skip(start); query.limit(count); return findList(query); } } src/main/java/com/yeshi/buwan/dao/juhe/iqiyi/VideoIqiyi2Dao.java
@@ -6,6 +6,8 @@ import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Repository; import java.util.List; @Repository public class VideoIqiyi2Dao extends MongodbBaseDao<VideoIqiyi2> { @@ -27,4 +29,11 @@ return findOne(query); } public List<VideoIqiyi2> listAll(int start, int count) { Query query = new Query(); query.skip(start); query.limit(count); return findList(query); } } src/main/java/com/yeshi/buwan/dto/mq/CMQResult.java
File was renamed from src/main/java/com/yeshi/buwan/dto/mq/CMQConsumeMsg.java @@ -1,10 +1,18 @@ package com.yeshi.buwan.dto.mq; //消息消费 public class CMQConsumeMsg { public class CMQResult { private String handler; private Object data; public CMQResult() { } public CMQResult(String handler, Object data) { this.handler = handler; this.data = data; } public String getHandler() { return handler; } src/main/java/com/yeshi/buwan/dto/mq/VideoExtraInfoChangeMQMsg.java
New file @@ -0,0 +1,51 @@ package com.yeshi.buwan.dto.mq; public class VideoExtraInfoChangeMQMsg { //添加 public final static String ACTION_ADD = "add"; //删除 public final static String ACTION_DELETE = "del"; //来源 public final static String TYPE_RESOURCE = "resource"; //分类 public final static String TYPE_CATEGORY = "category"; private String type; private String videoId; private String action; public VideoExtraInfoChangeMQMsg(String type, String videoId, String action) { this.type = type; this.videoId = videoId; this.action = action; } public VideoExtraInfoChangeMQMsg() { } public String getType() { return type; } public void setType(String type) { this.type = type; } public String getVideoId() { return videoId; } public void setVideoId(String videoId) { this.videoId = videoId; } public String getAction() { return action; } public void setAction(String action) { this.action = action; } } src/main/java/com/yeshi/buwan/funtv/FunTVUtil2.java
@@ -491,4 +491,9 @@ } public void update(int channelId,Date startTime,Date endTime){ } } src/main/java/com/yeshi/buwan/iqiyi/util/IqiyiUtil2.java
@@ -174,7 +174,7 @@ switch (channelId) { case IqiYiNewAPI.TYPE_DIANYING: updateAlbum(channelId, false, startTime, endTime); updateAlbum(channelId, true, startTime, endTime); break; case IqiYiNewAPI.TYPE_DIANSHIJU: case IqiYiNewAPI.TYPE_DONGMAN: src/main/java/com/yeshi/buwan/job/video/FunTV2VideoUpdate.java
@@ -31,6 +31,12 @@ private FunTV2Service funTV2Service; /** * 更新最近几天的视频 * @param param 天数 * @return * @throws Exception */ @XxlJob("video-update-funtv2-updateLatestVideo") public ReturnT<String> updateLatestVideo(String param) throws Exception { int d = 1; @@ -55,6 +61,12 @@ } /** * 根据专辑ID更新 * @param param * @return * @throws Exception */ @XxlJob("video-update-funtv2-updateAlbumById") public ReturnT<String> updateAlbumById(String param) throws Exception { FunTVAlbum2 album2 = FunTVNewApi.getAlbumsDetail(param); src/main/java/com/yeshi/buwan/job/video/Iqiyi2VideoUpdate.java
@@ -54,4 +54,16 @@ } @XxlJob("video-update-iqiyi-updateVideoByChannelId") public ReturnT<String> updateVideoByChannelId(String param) throws Exception { String[] params = param.split(","); for (String p : params) { int channelId = Integer.parseInt(p); iqiyiUtil2.updateTodayAlbumAndVideo(channelId); logger.info("更新当天专辑:" + channelId); } return ReturnT.SUCCESS; } } src/main/java/com/yeshi/buwan/service/imp/CategoryVideoService.java
@@ -3,6 +3,8 @@ import com.yeshi.buwan.dao.CategoryVideoDao; import com.yeshi.buwan.dao.ResourceVideoDao; import com.yeshi.buwan.domain.*; import com.yeshi.buwan.dto.mq.VideoExtraInfoChangeMQMsg; import com.yeshi.buwan.util.mq.CMQManager; import org.hibernate.HibernateException; import org.hibernate.Query; import org.hibernate.Session; @@ -26,6 +28,7 @@ cv.setVideo(new VideoInfo(videoId)); cv.setVideoType(new VideoType(categoryId)); categoryVideoDao.save(cv); CMQManager.getInstance().addVideoExtraInfoChanged(new VideoExtraInfoChangeMQMsg(VideoExtraInfoChangeMQMsg.TYPE_CATEGORY, videoId, VideoExtraInfoChangeMQMsg.ACTION_ADD)); } } src/main/java/com/yeshi/buwan/service/imp/ResourceVideoService.java
@@ -4,6 +4,7 @@ import java.util.List; import com.yeshi.buwan.domain.VideoResource; import com.yeshi.buwan.dto.mq.VideoExtraInfoChangeMQMsg; import com.yeshi.buwan.util.mq.CMQManager; import org.hibernate.HibernateException; import org.hibernate.Query; @@ -16,6 +17,8 @@ import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.domain.VideoInfo; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; @Service public class ResourceVideoService { @@ -65,6 +68,19 @@ return getResourceList(videoInfoList); } /** * 根据视频ID与ResourceId查询 * * @param videoId * @param sourceId * @return */ public long countByVideoIdAndResourceId(String videoId, long sourceId) { return resourceVideoDao.countByVideoIdAndResourceId(videoId, sourceId + ""); } public void save(ResourceVideo resourceVideo) { resourceVideoDao.save(resourceVideo); } @@ -72,16 +88,13 @@ public void addVideoResource(String videoId, String resourceId) { ResourceVideo rv = resourceVideoDao.selectByVideoIdAndResourceId(videoId, resourceId); if (rv == null) { System.out.println(videoId + ":" + resourceId); rv = new ResourceVideo(); rv.setResource(new VideoResource(resourceId + "")); rv.setVideo(new VideoInfo(videoId + "")); resourceVideoDao.save(rv); CMQManager.getInstance().addVideoExtraInfoChanged(new VideoExtraInfoChangeMQMsg(VideoExtraInfoChangeMQMsg.TYPE_RESOURCE, videoId, VideoExtraInfoChangeMQMsg.ACTION_ADD)); } } public void delete(ResourceVideo rv) { resourceVideoDao.delete(rv); } @Transactional @@ -90,6 +103,7 @@ if (resourceVideo != null) resourceVideoDao.delete(resourceVideo); CMQManager.getInstance().addVideoResourceDeleteMsg(videoId); CMQManager.getInstance().addVideoExtraInfoChanged(new VideoExtraInfoChangeMQMsg(VideoExtraInfoChangeMQMsg.TYPE_RESOURCE, videoId, VideoExtraInfoChangeMQMsg.ACTION_DELETE)); } src/main/java/com/yeshi/buwan/service/imp/VideoInfoService.java
@@ -89,7 +89,7 @@ } } public List<VideoDetailInfo> getVideoDetailList(String videoid, VideoResource vr,int page,int pageSize) { public List<VideoDetailInfo> getVideoDetailList(String videoid, VideoResource vr, int page, int pageSize) { VideoInfo find = videoInfoDao.find(VideoInfo.class, videoid); if (find != null) { if (vr.getName().equals("AcFun")) { @@ -185,9 +185,7 @@ public List<VideoInfo> listByVideoIds(List<String> videoIds) { return videoInfoDao.listByVideoIds(videoIds); } public void hiddenVideo(String id) { @@ -198,4 +196,10 @@ } } //从新统计 public void statisticVideoExtraInfo(String videoId) { videoInfoDao.statisticVideoExtraInfo(videoId); } } src/main/java/com/yeshi/buwan/service/imp/juhe/FunTV2ServiceImpl.java
@@ -257,25 +257,8 @@ VideoFunTV2 videoFunTV2 = videoFunTV2Dao.selectByMediaId(mediaId); if (videoFunTV2 == null) return; List<ResourceVideo> resourceVideoList = resourceVideoService.getResourceList(videoFunTV2.getVideoId() + ""); for (int i = 0; i < resourceVideoList.size(); i++) { ResourceVideo rv = resourceVideoList.get(i); if (rv.getResource() != null && Integer.parseInt(rv.getResource().getId()) == FunTVUtil2.RESOURCE_ID) { //删除资源映射 resourceVideoService.delete(rv); resourceVideoList.remove(i); i--; } } if (resourceVideoList.size() == 0) { //隐藏视频 videoInfoService.hiddenVideo(videoFunTV2.getVideoId() + ""); //更新搜索引擎 CMQManager.getInstance().addSolrMsg(videoFunTV2.getVideoId() + ""); } videoFunTV2Dao.delete(videoFunTV2.getVideoId()); resourceVideoService.delete(videoFunTV2.getVideoId()+"",FunTVUtil2.RESOURCE_ID+""); //删除媒体与媒体对应的视频 funTVAlbum2Dao.delete(mediaId); List<FunTVVideo2> video2List = listAllEpisodeList(mediaId); src/main/java/com/yeshi/buwan/service/imp/juhe/FunTVService.java
@@ -40,6 +40,7 @@ @Resource private FunTVAlbumDao funTVAlbumDao; @Resource private FunTVAlbumVideoMapDao funTVAlbumVideoMapDao; @@ -63,6 +64,9 @@ @Resource private VideoManager videoManager; @Resource private ResourceVideoService resourceVideoService; static Logger logger = Logger.getLogger(FunTVService.class); @@ -472,7 +476,9 @@ if (vft != null && vft.getAid() != null) {// 按节目查找视频 String aid = vft.getAid(); FunTVAlbum album = funTVAlbumDao.get(aid); if (album == null) { resourceVideoService.delete(videoid, FunTVUtil.RESOURCE_ID + ""); } final String type = album.getVideoType(); Sort.Direction sort = Sort.Direction.ASC; if (type != null) src/main/java/com/yeshi/buwan/util/HtmlToolUtil.java
New file @@ -0,0 +1,35 @@ package com.yeshi.buwan.util; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; public class HtmlToolUtil { //解析SQLYog工具导出的html内容 public static List<String[]> parseSQLYogExportHtmData(String path, int columns) { List<String[]> contentList = new ArrayList<>(); try { Document doc = Jsoup.parse(new File(path), "UTF-8"); Elements els = doc.getElementsByTag("tr"); for (int i = 1; i < els.size(); i++) { Elements tds = els.get(i).getElementsByTag("td"); String[] contents = new String[columns]; for (int j = 0; j < columns; j++) { String content = tds.get(j).html().trim(); contents[j] = content; } contentList.add(contents); } } catch (IOException e) { e.printStackTrace(); } return contentList; } } src/main/java/com/yeshi/buwan/util/SpringContext.java
@@ -2,12 +2,11 @@ import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.domain.VideoInfo; import com.yeshi.buwan.dto.mq.CMQConsumeMsg; import com.yeshi.buwan.dto.mq.CMQResult; import com.yeshi.buwan.dto.mq.FunTVAlbum2MQMsg; import com.yeshi.buwan.dto.mq.IqiyiAlbum2MQMsg; import com.yeshi.buwan.dto.mq.SolrVideoMQMsg; import com.yeshi.buwan.funtv.entity.FunTVAlbum2; import com.yeshi.buwan.funtv.entity.FunTVVideo2; import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2; import com.yeshi.buwan.service.imp.JobThreadExecutorServiceImpl; import com.yeshi.buwan.service.imp.ResourceVideoService; @@ -33,7 +32,6 @@ @Component public class SpringContext implements ApplicationListener<ContextRefreshedEvent> { @Resource private VideoInfoService videoInfoService; @@ -48,7 +46,6 @@ @Resource private ResourceVideoService resourceVideoService; private static boolean isInited = false; @@ -157,9 +154,9 @@ @Override public void run() { while (true) { List<CMQConsumeMsg> cmqMsgList = CMQManager.getInstance().consumeVideoResourceDeleteMsg(16); List<CMQResult> cmqMsgList = CMQManager.getInstance().consumeVideoResourceDeleteMsg(16); if (cmqMsgList != null) for (CMQConsumeMsg msg : cmqMsgList) { for (CMQResult msg : cmqMsgList) { try { String videoId = msg.getData() + ""; //查询资源列表 src/main/java/com/yeshi/buwan/util/mq/CMQManager.java
@@ -1,10 +1,8 @@ package com.yeshi.buwan.util.mq; import com.google.gson.Gson; import com.qcloud.cmq.Message; import com.yeshi.buwan.dto.mq.CMQConsumeMsg; import com.yeshi.buwan.dto.mq.FunTVAlbum2MQMsg; import com.yeshi.buwan.dto.mq.IqiyiAlbum2MQMsg; import com.yeshi.buwan.dto.mq.SolrVideoMQMsg; import com.yeshi.buwan.dto.mq.*; import java.util.ArrayList; import java.util.List; @@ -24,6 +22,12 @@ //删除视频资源 public static String QUEUENAME_VIDEO_RESOURCE_DELETE = "buwan-video-resource-delete"; //更新视频附加信息 public static String QUEUENAME_UPDATE_VIDEO_EXTRAINFO = "buwan-video-video-update-extrainfo"; private static String TOPIC_VIDEO_EXTRAINFO = "buwan_topic_video_extrainfo_change"; static { cmqUtil = CMQUtil.getInstance(secretId, secretKey); @@ -32,6 +36,18 @@ cmqUtil.createQueue(QUEUENAME_VIDEO_UPDATE_IQIYI_2, 1024 * 1024); cmqUtil.createQueue(QUEUENAME_VIDEO_UPDATE_FUNTV_2, 1024 * 1024); cmqUtil.createQueue(QUEUENAME_VIDEO_RESOURCE_DELETE, 1024 * 1024); cmqUtil.createQueue(QUEUENAME_UPDATE_VIDEO_EXTRAINFO, 1024 * 1024); //创建主题 cmqUtil.createTopic(TOPIC_VIDEO_EXTRAINFO); //订阅主题 try { List<String> filters = new ArrayList<>(); filters.add("resource"); filters.add("category"); cmqUtil.subscribeTopic(TOPIC_VIDEO_EXTRAINFO, TOPIC_VIDEO_EXTRAINFO + "#" + QUEUENAME_UPDATE_VIDEO_EXTRAINFO, QUEUENAME_UPDATE_VIDEO_EXTRAINFO, filters); } catch (Exception e) { } } public static CMQManager getInstance() { @@ -134,12 +150,12 @@ cmqUtil.sendMsg(QUEUENAME_VIDEO_RESOURCE_DELETE, videoId); } public List<CMQConsumeMsg> consumeVideoResourceDeleteMsg(int count) { List<CMQConsumeMsg> list = new ArrayList<>(); public List<CMQResult> consumeVideoResourceDeleteMsg(int count) { List<CMQResult> list = new ArrayList<>(); List<Message> msgList = cmqUtil.recieveMsg(count, QUEUENAME_VIDEO_RESOURCE_DELETE); if (msgList != null) for (Message msg : msgList) { CMQConsumeMsg mm = new CMQConsumeMsg(); CMQResult mm = new CMQResult(); mm.setHandler(msg.receiptHandle); mm.setData(msg.msgBody); list.add(mm); @@ -153,4 +169,32 @@ } /** * 视频附加信息改变 */ public void addVideoExtraInfoChanged(VideoExtraInfoChangeMQMsg msg) { List<String> list = new ArrayList<>(); list.add(msg.getType()); cmqUtil.publishTopicMessage(TOPIC_VIDEO_EXTRAINFO, new Gson().toJson(msg), list); } public List<CMQResult> consumeUpdateVideoExtraInfoMsg(int count) { List<CMQResult> list = new ArrayList<>(); List<Message> msgList = cmqUtil.recieveMsg(count, QUEUENAME_UPDATE_VIDEO_EXTRAINFO); if (msgList != null) for (Message msg : msgList) { CMQResult mm = new CMQResult(); mm.setHandler(msg.receiptHandle); mm.setData(new Gson().fromJson(msg.msgBody, VideoExtraInfoChangeMQMsg.class)); list.add(mm); } return list; } public void deleteUpdateVideoExtraInfoMsg(String handler) { cmqUtil.deleteMsg(QUEUENAME_UPDATE_VIDEO_EXTRAINFO, handler); } } src/main/java/com/yeshi/buwan/util/mq/CMQUtil.java
@@ -8,340 +8,381 @@ //腾讯CMQ消息 public class CMQUtil { private static CMQUtil cmqUtil; private static CMQUtil cmqUtil; public static CMQUtil getInstance(String secretId, String secretKey) { if (cmqUtil == null) { cmqUtil = new CMQUtil(); cmqUtil.init(secretId, secretKey); } return cmqUtil; } public static CMQUtil getInstance(String secretId, String secretKey) { if (cmqUtil == null) { cmqUtil = new CMQUtil(); cmqUtil.init(secretId, secretKey); } return cmqUtil; } private String secretId = ""; private String secretKey = ""; // 内网 http://cmq-queue-gz.api.tencentyun.com // 外网 http://cmq-queue-gz.api.qcloud.com private static String endpoint = "http://cmq-queue-gz.api.qcloud.com"; private static String topicEndPoint = "https://cmq-topic-gz.api.qcloud.com"; // private static String endpoint = // "http://cmq-queue-gz.api.tencentyun.com"; private String secretId = ""; private String secretKey = ""; // 内网 http://cmq-queue-gz.api.tencentyun.com // 外网 http://cmq-queue-gz.api.qcloud.com private static String endpoint = "http://cmq-queue-gz.api.qcloud.com"; private static String topicEndPoint = "https://cmq-topic-gz.api.qcloud.com"; // private static String endpoint = // "http://cmq-queue-gz.api.tencentyun.com"; private Account account; private Account topicAccount; private Account account; private Account topicAccount; static { // if (SystemUtil.getSystemType() == SystemUtil.SYSTEM_LINUX) // endpoint = "http://cmq-queue-gz.api.tencentyun.com"; // else endpoint = "http://cmq-queue-gz.api.qcloud.com"; } static { // if (SystemUtil.getSystemType() == SystemUtil.SYSTEM_LINUX) // endpoint = "http://cmq-queue-gz.api.tencentyun.com"; // else endpoint = "http://cmq-queue-gz.api.qcloud.com"; } public void init(String secretId, String secretKey) { this.secretId = secretId; this.secretKey = secretKey; account = new Account(endpoint, this.secretId, this.secretKey); topicAccount = new Account(topicEndPoint, this.secretId, this.secretKey); } public void init(String secretId, String secretKey) { this.secretId = secretId; this.secretKey = secretKey; account = new Account(endpoint, this.secretId, this.secretKey); topicAccount = new Account(topicEndPoint, this.secretId, this.secretKey); } public boolean existQueue(String queueName) { public boolean existQueue(String queueName) { ArrayList<String> vtQueue = new ArrayList<String>(); try { int totalCount = account.listQueue(queueName, 0, 1, vtQueue); if (totalCount <= 0) return false; } catch (Exception e) { e.printStackTrace(); } if (vtQueue.size() > 0 && vtQueue.get(0).equalsIgnoreCase(queueName)) return true; return false; } ArrayList<String> vtQueue = new ArrayList<String>(); try { int totalCount = account.listQueue(queueName, 0, 1, vtQueue); if (totalCount <= 0) return false; } catch (Exception e) { e.printStackTrace(); } if (vtQueue.size() > 0 && vtQueue.get(0).equalsIgnoreCase(queueName)) return true; return false; } // 创建队列 public boolean createQueue(String queueName) { // 创建队列 public boolean createQueue(String queueName) { QueueMeta meta = new QueueMeta(); meta.pollingWaitSeconds = 10; meta.visibilityTimeout = 5 * 60;// 消息可见性超时 meta.maxMsgSize = 65536; meta.msgRetentionSeconds = 345600; try { account.createQueue(queueName, meta); return true; } catch (Exception e) { return false; } } QueueMeta meta = new QueueMeta(); meta.pollingWaitSeconds = 10; meta.visibilityTimeout = 5 * 60;// 消息可见性超时 meta.maxMsgSize = 65536; meta.msgRetentionSeconds = 345600; try { account.createQueue(queueName, meta); return true; } catch (Exception e) { return false; } } // 创建队列 public boolean createQueue(String queueName, int maxMsgSize) { // 创建队列 public boolean createQueue(String queueName, int maxMsgSize) { QueueMeta meta = new QueueMeta(); meta.pollingWaitSeconds = 10; meta.visibilityTimeout = 5 * 60;// 消息可见性超时 meta.maxMsgSize = maxMsgSize; meta.msgRetentionSeconds = 345600; try { account.createQueue(queueName, meta); return true; } catch (Exception e) { return false; } } QueueMeta meta = new QueueMeta(); meta.pollingWaitSeconds = 10; meta.visibilityTimeout = 5 * 60;// 消息可见性超时 meta.maxMsgSize = maxMsgSize; meta.msgRetentionSeconds = 345600; try { account.createQueue(queueName, meta); return true; } catch (Exception e) { return false; } } /** * 指定参数创建队列 * * @param queueName * @param pollingWaitSeconds * -长轮训等待时间 * @param visibilityTimeout * -消息消费后再次可见的时间 * @return */ public boolean createQueue(String queueName, int pollingWaitSeconds, int visibilityTimeout) { /** * 指定参数创建队列 * * @param queueName * @param pollingWaitSeconds -长轮训等待时间 * @param visibilityTimeout -消息消费后再次可见的时间 * @return */ public boolean createQueue(String queueName, int pollingWaitSeconds, int visibilityTimeout) { QueueMeta meta = new QueueMeta(); meta.pollingWaitSeconds = pollingWaitSeconds; meta.visibilityTimeout = visibilityTimeout;// 消息可见性超时 meta.maxMsgSize = 65536; meta.msgRetentionSeconds = 345600; try { account.createQueue(queueName, meta); return true; } catch (Exception e) { return false; } } QueueMeta meta = new QueueMeta(); meta.pollingWaitSeconds = pollingWaitSeconds; meta.visibilityTimeout = visibilityTimeout;// 消息可见性超时 meta.maxMsgSize = 65536; meta.msgRetentionSeconds = 345600; try { // if (account.getQueue(queueName) != null) account.createQueue(queueName, meta); return true; } catch (Exception e) { return false; } } // 删除队列 public boolean deleteQueue(String queueName) { try { account.deleteQueue(queueName); return true; } catch (Exception e) { return false; } } // 删除队列 public boolean deleteQueue(String queueName) { try { account.deleteQueue(queueName); return true; } catch (Exception e) { return false; } } // 获取队列列表 public List<String> getQueueNameList(String key) { account = new Account(endpoint, this.secretId, this.secretKey); ArrayList<String> vtQueue = new ArrayList<String>(); try { int totalCount = account.listQueue(key, 0, 100, vtQueue); } catch (Exception e) { e.printStackTrace(); } return vtQueue; } // 获取队列列表 public List<String> getQueueNameList(String key) { account = new Account(endpoint, this.secretId, this.secretKey); ArrayList<String> vtQueue = new ArrayList<String>(); try { int totalCount = account.listQueue(key, 0, 100, vtQueue); } catch (Exception e) { e.printStackTrace(); } return vtQueue; } // 获取队列 public Queue getQueue(String queueName) { account = new Account(endpoint, this.secretId, this.secretKey); Queue queue = account.getQueue(queueName); return queue; } // 获取队列 public Queue getQueue(String queueName) { account = new Account(endpoint, this.secretId, this.secretKey); Queue queue = account.getQueue(queueName); return queue; } // 获取队列属性 public QueueMeta getQueueAtrribute(String queueName) { Queue queue = account.getQueue(queueName); QueueMeta meta2 = null; try { meta2 = queue.getQueueAttributes(); } catch (Exception e) { e.printStackTrace(); } return meta2; } // 获取队列属性 public QueueMeta getQueueAtrribute(String queueName) { Queue queue = account.getQueue(queueName); QueueMeta meta2 = null; try { meta2 = queue.getQueueAttributes(); } catch (Exception e) { e.printStackTrace(); } return meta2; } // 发送消息 public String sendMsg(String queueName, String msg) { try { Queue queue = getQueue(queueName); String msgId = queue.sendMessage(msg); return msgId; } catch (Exception e) { e.printStackTrace(); } return null; } // 发送消息 public String sendMsg(String queueName, String msg) { try { Queue queue = getQueue(queueName); String msgId = queue.sendMessage(msg); return msgId; } catch (Exception e) { e.printStackTrace(); } return null; } // 消费消息 public Message recieveMsg(String queueName) { try { Queue queue = getQueue(queueName); Message msg = queue.receiveMessage(10); return msg; } catch (Exception e) { e.printStackTrace(); } return null; } // 消费消息 public Message recieveMsg(String queueName) { try { Queue queue = getQueue(queueName); Message msg = queue.receiveMessage(10); return msg; } catch (Exception e) { e.printStackTrace(); } return null; } /** * 消费消息 * * @param count * 1-16 * @param queueName * 队列名字 * @return */ public List<Message> recieveMsg(int count, String queueName) { Queue queue = getQueue(queueName); /** * 消费消息 * * @param count 1-16 * @param queueName 队列名字 * @return */ public List<Message> recieveMsg(int count, String queueName) { Queue queue = getQueue(queueName); if (queue == null) { return null; } if (queue == null) { return null; } List<Message> msgList = null; try { msgList = queue.batchReceiveMessage(count, 20); return msgList; } catch (Exception e) { if (e.getMessage() != null && !e.getMessage().contains("no message")) e.printStackTrace(); } return null; } List<Message> msgList = null; try { msgList = queue.batchReceiveMessage(count, 20); return msgList; } catch (Exception e) { if (e.getMessage() != null && !e.getMessage().contains("no message")) e.printStackTrace(); } return null; } public List<Message> recieveMsg(int count, String queueName, int waitSeconds) { Queue queue = getQueue(queueName); List<Message> msgList = null; try { msgList = queue.batchReceiveMessage(count, waitSeconds); return msgList; } catch (Exception e) { } return null; } public List<Message> recieveMsg(int count, String queueName, int waitSeconds) { Queue queue = getQueue(queueName); List<Message> msgList = null; try { msgList = queue.batchReceiveMessage(count, waitSeconds); return msgList; } catch (Exception e) { } return null; } // 删除消息 public boolean deleteMsg(String queueName, String receiptHandle) { try { Queue queue = getQueue(queueName); queue.deleteMessage(receiptHandle); return true; } catch (Exception e) { e.printStackTrace(); } return false; } // 删除消息 public boolean deleteMsg(String queueName, String receiptHandle) { try { Queue queue = getQueue(queueName); queue.deleteMessage(receiptHandle); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 订阅消息相关 */ /** * 订阅消息相关 */ /** * 创建订阅主题 * * @param topicName-主题名称 * @param maxMsgSize-消息最大长度 * @param filterType-过滤类型 * @return */ public boolean createTopic(String topicName, int maxMsgSize, int filterType) { try { topicAccount.createTopic(topicName, maxMsgSize, filterType); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 创建订阅主题 * * @param topicName-主题名称 * @param maxMsgSize-消息最大长度 * @param filterType-过滤类型 * @return */ public boolean createTopic(String topicName, int maxMsgSize, int filterType) { try { // if (topicAccount.getTopic(topicName) == null) topicAccount.createTopic(topicName, maxMsgSize, filterType); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 创建默认参数的主题 * * @param topicName * @return */ public boolean createTopic(String topicName) { try { topicAccount.createTopic(topicName, 65536); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 创建默认参数的主题 * * @param topicName * @return */ public boolean createTopic(String topicName) { try { if (topicAccount.getTopic(topicName) == null) topicAccount.createTopic(topicName, 65536); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 订阅主题 * * @param topicName-主题名称 * @param subscriptionName-订阅名称 * @param queueName-接受消息的队列名称 * @return */ public boolean subscribeTopic(String topicName, String subscriptionName, String queueName) { try { topicAccount.createSubscribe(topicName, subscriptionName, queueName, "queue"); } catch (Exception e) { e.printStackTrace(); } return false; } /** * 订阅主题 * * @param topicName-主题名称 * @param subscriptionName-订阅名称 * @param queueName-接受消息的队列名称 * @return */ public boolean subscribeTopic(String topicName, String subscriptionName, String queueName) { try { topicAccount.createSubscribe(topicName, subscriptionName, queueName, "queue"); } catch (Exception e) { e.printStackTrace(); } return false; } /** * 删除订阅 * * @param topicName * @param subscriptionName * @return */ public boolean deleteSubscribeTopic(String topicName, String subscriptionName) { try { topicAccount.deleteSubscribe(topicName, subscriptionName); return true; } catch (Exception e) { e.printStackTrace(); } /** * 订阅 * * @param topicName * @param subscriptionName * @param queueName * @param filterTag * @return */ public boolean subscribeTopic(final String topicName, final String subscriptionName, String queueName, final List<String> filterTag) { try { topicAccount.createSubscribe(topicName, subscriptionName, queueName, "queue", filterTag, null, null, null); return true; } catch (Exception e) { e.printStackTrace(); } return false; } return false; } /** * 删除订阅 * * @param topicName * @param subscriptionName * @return */ public boolean deleteSubscribeTopic(String topicName, String subscriptionName) { try { topicAccount.deleteSubscribe(topicName, subscriptionName); return true; } catch (Exception e) { e.printStackTrace(); } /** * 发布订阅消息 * * @param topicName * @param message * @return */ public boolean publishTopicMessage(String topicName, String message) { try { Topic topic = topicAccount.getTopic(topicName); if (topic == null) return false; topic.publishMessage(message); return true; } catch (Exception e) { e.printStackTrace(); } return false; } return false; } /** * 批量发布消息 * * @param topicName * @param msgList * @return */ public boolean batchPublishTopicMessage(String topicName, List<String> msgList) { try { Topic topic = topicAccount.getTopic(topicName); if (topic == null) return false; topic.batchPublishMessage(msgList); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 发布订阅消息 * * @param topicName * @param message * @return */ public boolean publishTopicMessage(String topicName, String message) { try { Topic topic = topicAccount.getTopic(topicName); if (topic == null) return false; topic.publishMessage(message); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 发布订阅消息 * @param topicName * @param message * @param tags * @return */ public boolean publishTopicMessage(String topicName, String message, List<String> tags) { try { Topic topic = topicAccount.getTopic(topicName); if (topic == null) return false; topic.publishMessage(message, tags, null); return true; } catch (Exception e) { e.printStackTrace(); } return false; } /** * 批量发布消息 * * @param topicName * @param msgList * @return */ public boolean batchPublishTopicMessage(String topicName, List<String> msgList) { try { Topic topic = topicAccount.getTopic(topicName); if (topic == null) return false; topic.batchPublishMessage(msgList); return true; } catch (Exception e) { e.printStackTrace(); } return false; } } src/test/java/com/hxh/spring/test/FunTV2.java
@@ -1,12 +1,19 @@ package com.hxh.spring.test; import com.google.gson.Gson; import com.yeshi.buwan.dao.juhe.funtv.FunTVAlbum2Dao; import com.yeshi.buwan.dao.juhe.funtv.FunTVVideo2Dao; import com.yeshi.buwan.dao.juhe.funtv.VideoFunTV2Dao; import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.funtv.FunTVNewApi; import com.yeshi.buwan.funtv.FunTVUtil2; import com.yeshi.buwan.funtv.entity.FunTVAlbum2; import com.yeshi.buwan.funtv.entity.FunTVShortVideo2; import com.yeshi.buwan.funtv.entity.FunTVVideo2; import com.yeshi.buwan.funtv.entity.VideoFunTV2; import com.yeshi.buwan.job.video.FunTV2VideoUpdate; import com.yeshi.buwan.service.imp.ResourceVideoService; import com.yeshi.buwan.service.imp.VideoInfoService; import com.yeshi.buwan.service.inter.juhe.FunTV2Service; import com.yeshi.buwan.vo.video.funtv.Funtv2ResultVO; import org.junit.Test; @@ -37,6 +44,15 @@ @Resource private FunTV2VideoUpdate funTV2VideoUpdate; @Resource private VideoInfoService videoInfoService; @Resource private VideoFunTV2Dao videoFunTV2Dao; @Resource private ResourceVideoService resourceVideoService; @Test public void test1() { long now = System.currentTimeMillis(); @@ -49,12 +65,14 @@ @Test public void syncAlbum() { Funtv2ResultVO vo = FunTVNewApi.getAlbums(1, 20, null, null, 5, 1); //1-电影 2-电视剧 3-动漫 4-综艺 5-少儿 int channelId = 5; Funtv2ResultVO vo = FunTVNewApi.getAlbums(1, 20, null, null, channelId, 1); int totalCount = vo.getCount(); int pageSize = 20; int totalPage = totalCount % pageSize == 0 ? totalCount / pageSize : totalCount / pageSize + 1; for (int p = 0; p < totalPage; p++) { Funtv2ResultVO result = FunTVNewApi.getAlbums(p + 1, pageSize, null, null, 5, 1); Funtv2ResultVO result = FunTVNewApi.getAlbums(p + 1, pageSize, null, null, channelId, 1); if (result != null) for (Serializable a : result.getList()) { FunTVAlbum2 album2 = (FunTVAlbum2) a; @@ -107,12 +125,22 @@ @Test public void addToVideoInfo() { List<FunTVAlbum2> album2List = funTVAlbum2Dao.listByChannelId(1, 300, 500); List<FunTVAlbum2> album2List = funTVAlbum2Dao.listByChannelId(3, 2000, 1000); for (FunTVAlbum2 album2 : album2List) { System.out.println(album2.getName()); List<FunTVVideo2> list = funTVVideo2Dao.listByMediaId(album2.getId(), 0, 2000); album2.setEpisodes(list); funTV2Service.addToVideoInfo(album2); try { funTV2Service.processAlbum(album2); } catch (Exception e) { e.printStackTrace(); } } } @Test public void addToVideoInfo1() { videoInfoService.statisticVideoExtraInfo(179210 + ""); } @@ -123,9 +151,23 @@ @Test public void test() { FunTVVideo2 video = funTVVideo2Dao.get(34027 + ""); System.out.println(video); public void addResource() { List<String> videoIds = new ArrayList<>(); List<VideoFunTV2> list = videoFunTV2Dao.listAll(0, 5000); for (VideoFunTV2 funTV2 : list) { long count = resourceVideoService.countByVideoIdAndResourceId(funTV2.getVideoId() + "", FunTVUtil2.RESOURCE_ID); if (count==0L) { videoIds.add(funTV2.getVideoId() + ""); } } //["7993072","7993075","7993076","7993077","7993078","7993079","7993081","7993082","7993083","7993084","7993085","7993086","7993088","7993089","7993090","7993094","7993095","7993096","7595177","8066187","8066194","8066195","8066196","8076894","7582863","7582643","7582653","7582876","7583418","7582690","7595550","7595195","7582913","7582989","982912","5625845","7582991","7334865","7761779","8077307","8077347","8077354","8077355","8077397","8077399","7728744","7674559","7728747","7726042","359250","7053989","7351834"] System.out.println(new Gson().toJson(videoIds)); } @Test public void offLine() { funTV2Service.offLineAlbum("565"); } } src/test/java/com/hxh/spring/test/Iqiyi2.java
@@ -2,15 +2,24 @@ import com.google.gson.Gson; import com.yeshi.buwan.dao.juhe.iqiyi.IqiyiAlbum2Dao; import com.yeshi.buwan.dao.juhe.iqiyi.VideoIqiyi2Dao; import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.funtv.FunTVUtil2; import com.yeshi.buwan.funtv.entity.VideoFunTV2; import com.yeshi.buwan.iqiyi.IqiYiNewAPI; import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2; import com.yeshi.buwan.iqiyi.entity.VideoIqiyi2; import com.yeshi.buwan.iqiyi.util.IqiyiUtil2; import com.yeshi.buwan.iqiyi.vo.IqiyiAlbumListResult; import com.yeshi.buwan.job.SMSJob; import com.yeshi.buwan.job.video.Iqiyi2VideoUpdate; import com.yeshi.buwan.query.Iqiyi2AlbumQuery; import com.yeshi.buwan.service.imp.CategoryVideoService; import com.yeshi.buwan.service.imp.ResourceVideoService; import com.yeshi.buwan.service.inter.juhe.Iqiyi2Service; import com.yeshi.buwan.util.FileUtil; import com.yeshi.buwan.util.StringUtil; import com.yeshi.buwan.util.TimeUtil; import com.yeshi.buwan.util.*; import com.yeshi.buwan.util.mq.CMQManager; import net.sf.json.JSONArray; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; @@ -18,13 +27,11 @@ import org.springframework.test.context.web.WebAppConfiguration; import javax.annotation.Resource; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.*; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Scanner; // @RunWith(SpringJUnit4ClassRunner.class) //使用junit4进行测试 @@ -44,10 +51,19 @@ @Resource private Iqiyi2VideoUpdate iqiyi2VideoUpdate; @Resource private CategoryVideoService categoryVideoService; @Resource private VideoIqiyi2Dao videoIqiyi2Dao; @Resource private ResourceVideoService resourceVideoService; @Test public void syncAll() { iqiyiUtil2.syncAllDianShiJu(); iqiyiUtil2.updateTodayAlbumAndVideo(IqiYiNewAPI.TYPE_DIANYING); } @Test @@ -113,9 +129,11 @@ @Test public void test6() { for(int i=0;i<10;i++) { List<Long> ids = iqiyiAlbum2Dao.listInvalid(1000*(i), 1000); saveToFile(ids); for (int i = 0; i < 10; i++) { List<Long> ids = iqiyiAlbum2Dao.listInvalid(0, 1000); for (long id : ids) iqiyiAlbum2Dao.delete(id); //saveToFile(ids); } } @@ -146,7 +164,7 @@ public void count() { // long count = iqiyiAlbum2Dao.countVideoByAid(253120001L); // System.out.println(count); IqiyiAlbum2 album2 = iqiyiAlbum2Dao.get(6603128446259700L); IqiyiAlbum2 album2 = iqiyiAlbum2Dao.get(730266600L); iqiyi2Service.addToVideoInfo(album2); } @@ -163,5 +181,70 @@ System.out.println(result); } @Test public void deleteAlbum() { try { Scanner scanner = new Scanner(new FileInputStream(new File("E:\\iqiyialbum_invalid.txt"))); while (scanner.hasNextLine()) { String text = scanner.nextLine(); if (!StringUtil.isNullOrEmpty(text)) { JSONArray array = JSONArray.fromObject(text); for (int i = 0; i < array.size(); i++) { iqiyi2Service.deleteByAid(array.optLong(i)); } } } scanner.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } } @Test public void addToMQ() { Iqiyi2AlbumQuery query = new Iqiyi2AlbumQuery(); query.setContentType(1); query.setFeatureAlbumId(0L); query.setChannelId(2); long count = iqiyiAlbum2Dao.countByAid(query); int page = (int) (count % 100 == 0 ? count / 100 : count / 100 + 1); for (int i = 0; i < page; i++) { List<IqiyiAlbum2> album2List = iqiyiAlbum2Dao.listByAid(query, IqiyiAlbum2.SORT_ID, i * 100, 100); for (IqiyiAlbum2 album2 : album2List) { System.out.println(album2.getName()); if (IqiyiUtil2.albumIsValid(album2)) { CMQManager.getInstance().addIqiyiAlbumUpdateMsg(album2.getId()); } } } } @Test public void addCategory() { List<String[]> list = HtmlToolUtil.parseSQLYogExportHtmData("C:\\Users\\Administrator\\Desktop\\爱奇艺分类缺失.htm", 2); for (String[] sts : list) { String videoId = sts[0]; String videoType = sts[1]; categoryVideoService.addCategoryVideo(videoId, Long.parseLong(videoType)); } } @Test public void addResources() { List<String> videoIds = new ArrayList<>(); List<VideoIqiyi2> list = videoIqiyi2Dao.listAll(0, 5000); for (VideoIqiyi2 video : list) { long count = resourceVideoService.countByVideoIdAndResourceId(video.getVideoId() + "", IqiyiUtil2.RESOURCE_ID); if (count == 0L) { videoIds.add(video.getVideoId() + ""); } } //["588808","466573","475946","328312","475934","359261","276279","322163","276768"] System.out.println(new Gson().toJson(videoIds)); } }