From a66b556036c2b3936a51fd7b7e54a204eb31dc14 Mon Sep 17 00:00:00 2001
From: admin <weikou2014>
Date: 星期五, 30 七月 2021 19:13:14 +0800
Subject: [PATCH] 短视频爬虫，短视频搜索
---
src/main/java/com/yeshi/buwan/util/SpringContext.java | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 100 insertions(+), 1 deletions(-)
diff --git a/src/main/java/com/yeshi/buwan/util/SpringContext.java b/src/main/java/com/yeshi/buwan/util/SpringContext.java
index f11e6a0..8b55c36 100644
--- a/src/main/java/com/yeshi/buwan/util/SpringContext.java
+++ b/src/main/java/com/yeshi/buwan/util/SpringContext.java
@@ -1,23 +1,33 @@
package com.yeshi.buwan.util;
+import com.alibaba.druid.pool.DruidDataSource;
+import com.yeshi.buwan.dao.juhe.iqiyi.VideoIqiyi2Dao;
import com.yeshi.buwan.domain.ResourceVideo;
import com.yeshi.buwan.domain.VideoInfo;
import com.yeshi.buwan.domain.VideoResource;
+import com.yeshi.buwan.domain.video.InternetSearchVideo;
import com.yeshi.buwan.dto.mq.*;
import com.yeshi.buwan.funtv.entity.FunTVAlbum2;
+import com.yeshi.buwan.iqiyi.IqiYiNewAPI;
import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2;
+import com.yeshi.buwan.iqiyi.entity.VideoIqiyi2;
+import com.yeshi.buwan.iqiyi.util.IqiyiUtil2;
import com.yeshi.buwan.pptv.entity.PPTVSeries;
import com.yeshi.buwan.service.imp.JobThreadExecutorServiceImpl;
import com.yeshi.buwan.service.imp.ResourceVideoService;
import com.yeshi.buwan.service.imp.VideoInfoService;
import com.yeshi.buwan.service.inter.juhe.FunTV2Service;
+import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService;
import com.yeshi.buwan.service.inter.juhe.Iqiyi2Service;
import com.yeshi.buwan.service.inter.juhe.PPTVService;
-import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager;
+import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager;
+import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager;
import com.yeshi.buwan.util.mq.CMQManager;
+import io.seata.rm.datasource.DataSourceProxy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationListener;
+import org.springframework.context.annotation.Bean;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.stereotype.Component;
@@ -62,6 +72,12 @@
}
}
+ @Bean
+ public DataSourceProxy dataSourceProxy(DruidDataSource druidDataSource) {
+ return new DataSourceProxy(druidDataSource);
+ }
+
+
private synchronized void onApplication(ContextRefreshedEvent context) {
if (!isInited) {
isInited = true;
@@ -79,6 +95,8 @@
doAddPPTVVideo();
doDeleteVideoResource();
doUpdateVideoExtraInfo();
+ doInternetSearchVideoUpdateJob();
+ doUpdateResourceVideoJob();
}
}
@@ -118,6 +136,87 @@
}
+ @Resource
+ private InternetSearchVideoService internetSearchVideoService;
+
+ @Resource
+ private SolrInternetSearchVideoDataManager solrInternetSearchVideoDataManager;
+
+ private void doInternetSearchVideoUpdateJob() {
+ for (int i = 0; i < 1; i++) {
+ new JobThreadExecutorServiceImpl().run(new Runnable() {
+ @Override
+ public void run() {
+ List<InternetSearchVideoMQMsg> solrMsgList = CMQManager.getInstance().consumeInternetSearchVideoUpdateMsg(16);
+ if (solrMsgList != null)
+ for (InternetSearchVideoMQMsg solrVideo : solrMsgList) {
+ try {
+ InternetSearchVideo video = internetSearchVideoService.selectByPrimaryKey(solrVideo.getId());
+ if (video != null)
+ solrInternetSearchVideoDataManager.saveOrUpdate(video);
+ else
+ solrInternetSearchVideoDataManager.deleteById(video.getId());
+ CMQManager.getInstance().deleteInternetSearchVideoUpdateMsg(solrVideo.getHandler());
+ } catch (Exception e) {
+ logger.error("鏇存柊鍏ㄧ綉鎼滄悳绱㈠紩鎿庡嚭閿�", e);
+ }
+ }
+ }
+ });
+ }
+ }
+
+
+ @Resource
+ private RedisManager redisManager;
+
+ @Resource
+ private VideoIqiyi2Dao videoIqiyi2Dao;
+
+ private void doUpdateResourceVideoJob() {
+ for (int i = 0; i < 1; i++) {
+ new JobThreadExecutorServiceImpl().run(new Runnable() {
+ @Override
+ public void run() {
+ List<UpdateResourceVideoMQMsg> msgList = CMQManager.getInstance().consumeUpdateResourceVideoMsg(16);
+ if (msgList != null)
+ for (UpdateResourceVideoMQMsg videoMQMsg : msgList) {
+ try {
+ if (videoMQMsg != null) {
+ String key = "resourcevideo-update-" + StringUtil.Md5(videoMQMsg.getResourceId() + "#" + videoMQMsg.getId());
+ //鍙互鏇存柊
+ if (StringUtil.isNullOrEmpty(redisManager.getCommonString(key))) {
+ switch (videoMQMsg.getResourceId()) {
+ case IqiyiUtil2
+ .RESOURCE_ID:
+ //redis鏌ヨ鏄惁鏇存柊杩�
+ IqiyiAlbum2 iqiyiAlbum2 = IqiYiNewAPI.getAlbumOrVideoDetail(Long.parseLong(videoMQMsg.getId()));
+ if (iqiyiAlbum2 == null) {
+ VideoIqiyi2 videoIqiyi2 = videoIqiyi2Dao.selectByIqiyiId(Long.parseLong(videoMQMsg.getId()));
+ if (videoIqiyi2 != null) {
+ //涓嬫灦
+ resourceVideoService.delete(videoIqiyi2.getVideoId() + "", videoMQMsg.getResourceId() + "");
+ }
+ }
+ //鍒犻櫎鐖卞鑹鸿祫婧�
+ iqiyi2Service.offlineIqiyiAlbum(Long.parseLong(videoMQMsg.getId()));
+ break;
+ }
+ //12灏忔椂鍐呬笉鍐嶆洿鏂�
+ redisManager.cacheCommonString(key, "1", 60 * 60 * 12);
+ }
+ }
+ } catch (Exception e) {
+ } finally {
+ CMQManager.getInstance().deleteUpdateResourceVideoMsg(videoMQMsg.getHandler());
+ }
+ }
+ }
+ });
+ }
+ }
+
+
private void doAddIqiyi2Video() {
new JobThreadExecutorServiceImpl().run(new Runnable() {
@Override
--
Gitblit v1.8.0