pom.xml
@@ -507,7 +507,7 @@ <dependency> <groupId>com.yeshi</groupId> <artifactId>utils</artifactId> <version>0.0.1-SNAPSHOT</version> <version>0.0.2-SNAPSHOT</version> </dependency>
File was deleted src/main/java/com/qcloud/cmq/CMQClient.java
File was deleted src/main/java/com/qcloud/cmq/CMQClientException.java
File was deleted src/main/java/com/qcloud/cmq/CMQHttp.java
File was deleted src/main/java/com/qcloud/cmq/CMQServerException.java
File was deleted src/main/java/com/qcloud/cmq/CMQTool.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONArray.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONException.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONObject.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONString.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONStringer.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONTokener.java
File was deleted src/main/java/com/qcloud/cmq/Json/JSONWriter.java
File was deleted src/main/java/com/qcloud/cmq/Message.java
File was deleted src/main/java/com/qcloud/cmq/Queue.java
File was deleted src/main/java/com/qcloud/cmq/QueueMeta.java
File was deleted src/main/java/com/qcloud/cmq/Subscription.java
File was deleted src/main/java/com/qcloud/cmq/SubscriptionMeta.java
File was deleted src/main/java/com/qcloud/cmq/Topic.java
File was deleted src/main/java/com/qcloud/cmq/TopicMeta.java
File was deleted src/main/java/com/yeshi/buwan/dao/juhe/funtv/VideoFunTVDao.java
@@ -29,4 +29,5 @@ return funTVList; } } src/main/java/com/yeshi/buwan/dao/juhe/iqiyi/VideoIqiyiDao.java
@@ -4,7 +4,20 @@ import com.yeshi.buwan.dao.base.BaseDao; import com.yeshi.buwan.iqiyi.entity.VideoIqiyi; import java.util.List; @Repository public class VideoIqiyiDao extends BaseDao<VideoIqiyi> { /** * 根据VideoId检索 * * @param videoId * @return */ public List<VideoIqiyi> listByVideoId(String videoId) { return list("from VideoIqiyi video where video.video.id=" + videoId); } } src/main/java/com/yeshi/buwan/service/imp/ResourceVideoService.java
@@ -3,8 +3,10 @@ import java.util.ArrayList; import java.util.List; import com.yeshi.buwan.dao.juhe.iqiyi.VideoIqiyiDao; import com.yeshi.buwan.domain.VideoResource; import com.yeshi.buwan.dto.mq.VideoExtraInfoChangeMQMsg; import com.yeshi.buwan.iqiyi.entity.VideoIqiyi; import com.yeshi.buwan.util.mq.CMQManager; import org.hibernate.HibernateException; import org.hibernate.Query; @@ -25,6 +27,7 @@ @Autowired private ResourceVideoDao resourceVideoDao; @SuppressWarnings("unchecked") public List<ResourceVideo> getResourceList(final List<VideoInfo> list) { @@ -100,10 +103,10 @@ @Transactional public void delete(String videoId, String resourceId) { ResourceVideo resourceVideo = resourceVideoDao.selectByVideoIdAndResourceId(videoId, resourceId + ""); if (resourceVideo != null) if (resourceVideo != null) { resourceVideoDao.delete(resourceVideo); CMQManager.getInstance().addVideoResourceDeleteMsg(videoId); CMQManager.getInstance().addVideoExtraInfoChanged(new VideoExtraInfoChangeMQMsg(VideoExtraInfoChangeMQMsg.TYPE_RESOURCE, videoId, VideoExtraInfoChangeMQMsg.ACTION_DELETE)); CMQManager.getInstance().addVideoExtraInfoChanged(new VideoExtraInfoChangeMQMsg(VideoExtraInfoChangeMQMsg.TYPE_RESOURCE, videoId, VideoExtraInfoChangeMQMsg.ACTION_DELETE)); } }
@@ -526,6 +526,8 @@ detailList.add(detail); return detailList; } }else{//删除来源 resourceVideoService.delete(videoid,FunTVUtil.RESOURCE_ID+""); } return null; src/main/java/com/yeshi/buwan/util/SpringContext.java
@@ -2,10 +2,7 @@ import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.domain.VideoInfo; import com.yeshi.buwan.dto.mq.CMQResult; import com.yeshi.buwan.dto.mq.FunTVAlbum2MQMsg; import com.yeshi.buwan.dto.mq.IqiyiAlbum2MQMsg; import com.yeshi.buwan.dto.mq.SolrVideoMQMsg; import com.yeshi.buwan.dto.mq.*; import com.yeshi.buwan.funtv.entity.FunTVAlbum2; import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2; import com.yeshi.buwan.service.imp.JobThreadExecutorServiceImpl; @@ -72,6 +69,7 @@ doAddIqiyi2Video(); doAddFunTV2Video(); doDeleteVideoResource(); doUpdateVideoExtraInfo(); } } @@ -79,22 +77,22 @@ new JobThreadExecutorServiceImpl().run(new Runnable() { @Override public void run() { List<SolrVideoMQMsg> solrMsgList = CMQManager.getInstance().consumeSolrMsg(16); if (solrMsgList != null) for (SolrVideoMQMsg solrVideo : solrMsgList) { try { VideoInfo videoInfo = videoInfoService.getVideoInfo(solrVideo.getId()); if (videoInfo != null) { if ("1".equalsIgnoreCase(videoInfo.getShow())) solrDataManager.saveOrUpdate(videoInfo); else solrDataManager.deleteById(videoInfo.getId()); } CMQManager.getInstance().deleteSolrMsg(solrVideo.getHandler()); } catch (Exception e) { List<SolrVideoMQMsg> solrMsgList = CMQManager.getInstance().consumeSolrMsg(16); if (solrMsgList != null) for (SolrVideoMQMsg solrVideo : solrMsgList) { try { VideoInfo videoInfo = videoInfoService.getVideoInfo(solrVideo.getId()); if (videoInfo != null) { if ("1".equalsIgnoreCase(videoInfo.getShow())) solrDataManager.saveOrUpdate(videoInfo); else solrDataManager.deleteById(videoInfo.getId()); } CMQManager.getInstance().deleteSolrMsg(solrVideo.getHandler()); } catch (Exception e) { } } } }); } @@ -104,22 +102,22 @@ new JobThreadExecutorServiceImpl().run(new Runnable() { @Override public void run() { List<IqiyiAlbum2MQMsg> iqiyiAlbumMsgList = CMQManager.getInstance().consumeIqiyiAlbumUpdateMsg(16); if (iqiyiAlbumMsgList != null) for (IqiyiAlbum2MQMsg iqiyiAlbum2MQMsg : iqiyiAlbumMsgList) { try { Long qikuID = iqiyiAlbum2MQMsg.getId(); IqiyiAlbum2 album2 = iqiyi2Service.selectAlbumById(qikuID); if (album2 != null) { iqiyi2Service.addToVideoInfo(album2); } CMQManager.getInstance().deleteIqiyiAlbumUpdateMsg(iqiyiAlbum2MQMsg.getHandler()); } catch (Exception e) { logger.error("爱奇艺专辑添加到视频出错：" + e.getMessage()); logger.error("ID:" + iqiyiAlbum2MQMsg.getId()); List<IqiyiAlbum2MQMsg> iqiyiAlbumMsgList = CMQManager.getInstance().consumeIqiyiAlbumUpdateMsg(16); if (iqiyiAlbumMsgList != null) for (IqiyiAlbum2MQMsg iqiyiAlbum2MQMsg : iqiyiAlbumMsgList) { try { Long qikuID = iqiyiAlbum2MQMsg.getId(); IqiyiAlbum2 album2 = iqiyi2Service.selectAlbumById(qikuID); if (album2 != null) { iqiyi2Service.addToVideoInfo(album2); } CMQManager.getInstance().deleteIqiyiAlbumUpdateMsg(iqiyiAlbum2MQMsg.getHandler()); } catch (Exception e) { logger.error("爱奇艺专辑添加到视频出错：" + e.getMessage()); logger.error("ID:" + iqiyiAlbum2MQMsg.getId()); } } } } }); } @@ -129,21 +127,21 @@ new JobThreadExecutorServiceImpl().run(new Runnable() { @Override public void run() { List<FunTVAlbum2MQMsg> funTVAlbum2MsgList = CMQManager.getInstance().consumeFunTVAlbumUpdateMsg(16); if (funTVAlbum2MsgList != null) for (FunTVAlbum2MQMsg funTVAlbum2MQMsg : funTVAlbum2MsgList) { try { String mediaId = funTVAlbum2MQMsg.getId(); FunTVAlbum2 album2 = funTV2Service.getAlbumDetail(mediaId); if (album2 != null) { funTV2Service.processAlbum(album2); } CMQManager.getInstance().deleteFunTVAlbumUpdateMsg(funTVAlbum2MQMsg.getHandler()); } catch (Exception e) { logger.error("风行专辑添加到视频出错：" + e.getMessage()); logger.error("ID:" + funTVAlbum2MQMsg.getId()); List<FunTVAlbum2MQMsg> funTVAlbum2MsgList = CMQManager.getInstance().consumeFunTVAlbumUpdateMsg(16); if (funTVAlbum2MsgList != null) for (FunTVAlbum2MQMsg funTVAlbum2MQMsg : funTVAlbum2MsgList) { try { String mediaId = funTVAlbum2MQMsg.getId(); FunTVAlbum2 album2 = funTV2Service.getAlbumDetail(mediaId); if (album2 != null) { funTV2Service.processAlbum(album2); } CMQManager.getInstance().deleteFunTVAlbumUpdateMsg(funTVAlbum2MQMsg.getHandler()); } catch (Exception e) { logger.error("风行专辑添加到视频出错：" + e.getMessage()); logger.error("ID:" + funTVAlbum2MQMsg.getId()); } } } }); @@ -153,26 +151,59 @@ new JobThreadExecutorServiceImpl().run(new Runnable() { @Override public void run() { while (true) { List<CMQResult> cmqMsgList = CMQManager.getInstance().consumeVideoResourceDeleteMsg(16); if (cmqMsgList != null) for (CMQResult msg : cmqMsgList) { try { String videoId = msg.getData() + ""; //查询资源列表 List<ResourceVideo> resourceVideoList = resourceVideoService.getResourceList(videoId); //隐藏视频 if (resourceVideoList == null || resourceVideoList.size() == 0) videoInfoService.hiddenVideo(videoId); //更新搜索引擎 CMQManager.getInstance().addSolrMsg(videoId); CMQManager.getInstance().deleteVideoResourceDeleteMsg(msg.getHandler()); } catch (Exception e) { logger.error("视频资源删除处理出错：" + e.getMessage()); logger.error("ID:" + msg.getData()); } List<CMQResult> cmqMsgList = CMQManager.getInstance().consumeVideoResourceDeleteMsg(16); if (cmqMsgList != null) for (CMQResult msg : cmqMsgList) { try { String videoId = msg.getData() + ""; //查询资源列表 List<ResourceVideo> resourceVideoList = resourceVideoService.getResourceList(videoId); //隐藏视频 if (resourceVideoList == null || resourceVideoList.size() == 0) videoInfoService.hiddenVideo(videoId); //更新搜索引擎 CMQManager.getInstance().addSolrMsg(videoId); CMQManager.getInstance().deleteVideoResourceDeleteMsg(msg.getHandler()); } catch (Exception e) { logger.error("视频资源删除处理出错：" + e.getMessage()); logger.error("ID:" + msg.getData()); } } } } }); } private void doUpdateVideoExtraInfo() { new JobThreadExecutorServiceImpl().run(new Runnable() { @Override public void run() { List<CMQResult> cmqMsgList = CMQManager.getInstance().consumeUpdateVideoExtraInfoMsg(16); if (cmqMsgList != null) for (CMQResult msg : cmqMsgList) { try { VideoExtraInfoChangeMQMsg videoExtraInfoChangeMQMsg = (VideoExtraInfoChangeMQMsg) msg.getData(); if (videoExtraInfoChangeMQMsg != null) { switch (videoExtraInfoChangeMQMsg.getType()) { case VideoExtraInfoChangeMQMsg.TYPE_RESOURCE: if (VideoExtraInfoChangeMQMsg.ACTION_DELETE.equalsIgnoreCase(videoExtraInfoChangeMQMsg.getAction())) {//删除视频源 CMQManager.getInstance().addVideoResourceDeleteMsg(videoExtraInfoChangeMQMsg.getVideoId()); } break; case VideoExtraInfoChangeMQMsg.TYPE_CATEGORY: break; } videoInfoService.statisticVideoExtraInfo(videoExtraInfoChangeMQMsg.getVideoId()); } CMQManager.getInstance().deleteUpdateVideoExtraInfoMsg(msg.getHandler()); } catch (Exception e) { logger.error("视频资源删除处理出错：" + e.getMessage()); logger.error("ID:" + msg.getData()); } } } });
@@ -3,6 +3,7 @@ import com.google.gson.Gson; import com.qcloud.cmq.Message; import com.yeshi.buwan.dto.mq.*; import com.yeshi.buwan.util.StringUtil; import java.util.ArrayList; import java.util.List; @@ -45,7 +46,7 @@ List<String> filters = new ArrayList<>(); filters.add("resource"); filters.add("category"); cmqUtil.subscribeTopic(TOPIC_VIDEO_EXTRAINFO, TOPIC_VIDEO_EXTRAINFO + "#" + QUEUENAME_UPDATE_VIDEO_EXTRAINFO, QUEUENAME_UPDATE_VIDEO_EXTRAINFO, filters); cmqUtil.subscribeTopic(TOPIC_VIDEO_EXTRAINFO, StringUtil.Md5(TOPIC_VIDEO_EXTRAINFO + "#" + QUEUENAME_UPDATE_VIDEO_EXTRAINFO), QUEUENAME_UPDATE_VIDEO_EXTRAINFO, filters); } catch (Exception e) { } } src/main/java/com/yeshi/buwan/util/mq/CMQUtil.java
@@ -298,7 +298,7 @@ final List<String> filterTag) { try { topicAccount.createSubscribe(topicName, subscriptionName, queueName, "queue", filterTag, null, null, null); filterTag, null, "EXPONENTIAL_DECAY_RETRY", "JSON"); return true; } catch (Exception e) { e.printStackTrace(); src/test/java/com/hxh/spring/test/FunTV2.java
@@ -1,21 +1,22 @@ package com.hxh.spring.test; import com.google.gson.Gson; import com.yeshi.buwan.dao.juhe.funtv.FunTVAlbum2Dao; import com.yeshi.buwan.dao.juhe.funtv.FunTVVideo2Dao; import com.yeshi.buwan.dao.juhe.funtv.VideoFunTV2Dao; import com.yeshi.buwan.dao.juhe.funtv.*; import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.dto.mq.VideoExtraInfoChangeMQMsg; import com.yeshi.buwan.funtv.FunTVNewApi; import com.yeshi.buwan.funtv.FunTVUtil; import com.yeshi.buwan.funtv.FunTVUtil2; import com.yeshi.buwan.funtv.entity.FunTVAlbum2; import com.yeshi.buwan.funtv.entity.FunTVShortVideo2; import com.yeshi.buwan.funtv.entity.FunTVVideo2; import com.yeshi.buwan.funtv.entity.VideoFunTV2; import com.yeshi.buwan.funtv.entity.*; import com.yeshi.buwan.iqiyi.entity.VideoIqiyi; import com.yeshi.buwan.job.video.FunTV2VideoUpdate; import com.yeshi.buwan.service.imp.ResourceVideoService; import com.yeshi.buwan.service.imp.VideoInfoService; import com.yeshi.buwan.service.inter.juhe.FunTV2Service; import com.yeshi.buwan.util.HtmlToolUtil; import com.yeshi.buwan.util.mq.CMQManager; import com.yeshi.buwan.vo.video.funtv.Funtv2ResultVO; import net.sf.json.JSONArray; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; @@ -125,7 +126,7 @@ @Test public void addToVideoInfo() { List<FunTVAlbum2> album2List = funTVAlbum2Dao.listByChannelId(3, 2000, 1000); List<FunTVAlbum2> album2List = funTVAlbum2Dao.listByChannelId(2, 0, 1000); for (FunTVAlbum2 album2 : album2List) { System.out.println(album2.getName()); List<FunTVVideo2> list = funTVVideo2Dao.listByMediaId(album2.getId(), 0, 2000); @@ -152,16 +153,25 @@ @Test public void addResource() { List<String> videoIds = new ArrayList<>(); List<VideoFunTV2> list = videoFunTV2Dao.listAll(0, 5000); for (VideoFunTV2 funTV2 : list) { long count = resourceVideoService.countByVideoIdAndResourceId(funTV2.getVideoId() + "", FunTVUtil2.RESOURCE_ID); if (count==0L) { videoIds.add(funTV2.getVideoId() + ""); } } //["7993072","7993075","7993076","7993077","7993078","7993079","7993081","7993082","7993083","7993084","7993085","7993086","7993088","7993089","7993090","7993094","7993095","7993096","7595177","8066187","8066194","8066195","8066196","8076894","7582863","7582643","7582653","7582876","7583418","7582690","7595550","7595195","7582913","7582989","982912","5625845","7582991","7334865","7761779","8077307","8077347","8077354","8077355","8077397","8077399","7728744","7674559","7728747","7726042","359250","7053989","7351834"] System.out.println(new Gson().toJson(videoIds)); // List<String> videoIds = new ArrayList<>(); // List<VideoFunTV2> list = videoFunTV2Dao.listAll(0, 5000); // for (VideoFunTV2 funTV2 : list) { // long count = resourceVideoService.countByVideoIdAndResourceId(funTV2.getVideoId() + "", FunTVUtil2.RESOURCE_ID); // if (count == 0L) { // videoIds.add(funTV2.getVideoId() + ""); // } // } // System.out.println(new Gson().toJson(videoIds)); // String json = "[\"7993076\",\"7993077\",\"7993078\"]"; // JSONArray array = JSONArray.fromObject(json); // for (int i = 0; i < array.size(); i++) { // resourceVideoService.addVideoResource(array.optString(i), FunTVUtil2.RESOURCE_ID + ""); // } // // CMQManager.getInstance().addVideoExtraInfoChanged(new VideoExtraInfoChangeMQMsg(VideoExtraInfoChangeMQMsg.TYPE_RESOURCE, "7993076", VideoExtraInfoChangeMQMsg.ACTION_ADD)); // resourceVideoService.addVideoResource("7993076", FunTVUtil2.RESOURCE_ID + ""); } @@ -170,4 +180,22 @@ funTV2Service.offLineAlbum("565"); } @Resource private VideoFunTVNewDao videoFunTVNewDao; @Test public void deleteOldFunTVVideo() { List<String[]> list = HtmlToolUtil.parseSQLYogExportHtmData("C:\\Users\\Administrator\\Desktop\\风行老版正片.htm", 3); for (String[] sts : list) { String videoId = sts[0]; resourceVideoService.delete(videoId, FunTVUtil.RESOURCE_ID+""); // List<VideoFunTV> list1 = videoFunTVNewDao.listByVideoId(Long.parseLong(videoId)); // // if (list1 != null) // for (VideoFunTV videoIqiyi : list1) // videoFunTVNewDao.delete(videoIqiyi.getId()); } } } src/test/java/com/hxh/spring/test/Iqiyi2.java
@@ -3,12 +3,15 @@ import com.google.gson.Gson; import com.yeshi.buwan.dao.juhe.iqiyi.IqiyiAlbum2Dao; import com.yeshi.buwan.dao.juhe.iqiyi.VideoIqiyi2Dao; import com.yeshi.buwan.dao.juhe.iqiyi.VideoIqiyiDao; import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.funtv.FunTVUtil2; import com.yeshi.buwan.funtv.entity.VideoFunTV2; import com.yeshi.buwan.iqiyi.IqiYiNewAPI; import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2; import com.yeshi.buwan.iqiyi.entity.VideoIqiyi; import com.yeshi.buwan.iqiyi.entity.VideoIqiyi2; import com.yeshi.buwan.iqiyi.util.IqiyiUtil; import com.yeshi.buwan.iqiyi.util.IqiyiUtil2; import com.yeshi.buwan.iqiyi.vo.IqiyiAlbumListResult; import com.yeshi.buwan.job.SMSJob; @@ -232,18 +235,40 @@ } } @Resource private VideoIqiyiDao videoIqiyiDao; @Test public void deleteOldIqiyiVideo() { List<String[]> list = HtmlToolUtil.parseSQLYogExportHtmData("C:\\Users\\Administrator\\Desktop\\爱奇艺老版正片.htm", 3); for (String[] sts : list) { String videoId = sts[0]; List<VideoIqiyi> list1 = videoIqiyiDao.listByVideoId(videoId); if (list1 != null) for (VideoIqiyi videoIqiyi : list1) videoIqiyiDao.delete(videoIqiyi); } } @Test public void addResources() { List<String> videoIds = new ArrayList<>(); List<VideoIqiyi2> list = videoIqiyi2Dao.listAll(0, 5000); for (VideoIqiyi2 video : list) { long count = resourceVideoService.countByVideoIdAndResourceId(video.getVideoId() + "", IqiyiUtil2.RESOURCE_ID); if (count == 0L) { videoIds.add(video.getVideoId() + ""); } // List<String> videoIds = new ArrayList<>(); // List<VideoIqiyi2> list = videoIqiyi2Dao.listAll(0, 5000); // for (VideoIqiyi2 video : list) { // long count = resourceVideoService.countByVideoIdAndResourceId(video.getVideoId() + "", IqiyiUtil2.RESOURCE_ID); // if (count == 0L) { // videoIds.add(video.getVideoId() + ""); // } // } String json = "[\"588808\",\"466573\",\"475946\",\"328312\",\"475934\",\"359261\",\"276279\",\"322163\",\"276768\"]"; JSONArray array = JSONArray.fromObject(json); for (int i = 0; i < array.size(); i++) { resourceVideoService.addVideoResource(array.optString(i), IqiyiUtil2.RESOURCE_ID + ""); } //["588808","466573","475946","328312","475934","359261","276279","322163","276768"] System.out.println(new Gson().toJson(videoIds)); }
