From a66b556036c2b3936a51fd7b7e54a204eb31dc14 Mon Sep 17 00:00:00 2001
From: admin <weikou2014>
Date: 星期五, 30 七月 2021 19:13:14 +0800
Subject: [PATCH] 短视频爬虫，短视频搜索
---
src/main/java/com/yeshi/buwan/controller/parser/HomeParser.java | 77 ++++++++++++++++++++++++++++++--------
1 files changed, 61 insertions(+), 16 deletions(-)
diff --git a/src/main/java/com/yeshi/buwan/controller/parser/HomeParser.java b/src/main/java/com/yeshi/buwan/controller/parser/HomeParser.java
index 4d60e6c..1c443a9 100644
--- a/src/main/java/com/yeshi/buwan/controller/parser/HomeParser.java
+++ b/src/main/java/com/yeshi/buwan/controller/parser/HomeParser.java
@@ -31,6 +31,7 @@
import com.yeshi.buwan.service.inter.juhe.PPTVService;
import com.yeshi.buwan.service.inter.juhe.YouKuService;
import com.yeshi.buwan.service.inter.recommend.HomeRecommendSpecialService;
+import com.yeshi.buwan.service.inter.recommend.HomeVideoService;
import com.yeshi.buwan.service.inter.video.VideoWatchHistoryService;
import com.yeshi.buwan.service.manager.VideoPlayStatisticManager;
import com.yeshi.buwan.tencent.TencentVideoUtil;
@@ -40,6 +41,7 @@
import com.yeshi.buwan.util.log.LoggerUtil;
import com.yeshi.buwan.util.log.UserActiveLogFactory;
import com.yeshi.buwan.util.log.VideoLogFactory;
+import com.yeshi.buwan.videos.bilibili.BilibiliUtil;
import com.yeshi.buwan.vo.AcceptData;
import com.yeshi.buwan.vo.video.VideoDetailVO;
import com.yeshi.buwan.youku.YouKuUtil;
@@ -71,6 +73,10 @@
private HomeAdService homeAdService;
@Resource
private HomeTypeService homeTypeService;
+
+ @Resource
+ private HomeVideoService homeVideoService;
+
@Resource
private DetailSystemConfigService configService;
@Resource
@@ -194,7 +200,7 @@
DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());
JSONObject object = new JSONObject();
- List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
+ List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
List<HomeType> list1 = homeTypeService.getHomeType(detailSystem.getId(), acceptData.getPlatform(), acceptData.getVersion(), resourceList,
CacheUtil.getMD5Long(resourceList), -1, special != null ? special.getDataKey() : null);
List<HomeType> list = new ArrayList<>();
@@ -301,7 +307,7 @@
long count = homeTypeService.countHomeType(acceptData.getDetailSystem().getId(), acceptData.getVersion(), dataKey);
- List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
+ List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
List<HomeType> homeTypeList = homeTypeService.getHomeTypeList(acceptData.getDetailSystem().getId(), acceptData.getVersion(), dataKey, Integer.parseInt(page), Integer.parseInt(pageSize));
if (homeTypeList != null)
@@ -310,11 +316,11 @@
long bigPictureCount = homeType.isBigPicture() ? 1 : 0;// homeTypeService.getHomeVideoListCount(homeType.getId(), resourceKey, resourceList, true);
- List<HomeVideo> homeVideoList = homeTypeService.getHomeVideoList(homeType.getId(), resourceKey, resourceList, null, 1, homeType.getNumber() + (int) bigPictureCount);
+ List<HomeVideo> homeVideoList = homeVideoService.getHomeVideoList(homeType.getId(), resourceKey, resourceList, null, 1, homeType.getNumber() + (int) bigPictureCount);
if (bigPictureCount > 0 && homeVideoList.size() > 0) {
homeVideoList.get(0).setBigPicture(true);
}
- long videoCount = homeTypeService.getHomeVideoListCount(homeType.getId(), resourceKey, resourceList, null);
+ long videoCount = homeVideoService.getHomeVideoListCount(homeType.getId(), resourceKey, resourceList, null);
if (videoCount - bigPictureCount > homeType.getNumber()) {
while ((videoCount - bigPictureCount) % homeType.getNumber() != 0) {
@@ -324,11 +330,33 @@
homeType.setCount((int) videoCount);
List<HomeVideo> tempHomeVideoList = new ArrayList<>();
tempHomeVideoList.addAll(homeVideoList);
- while (tempHomeVideoList != null && (tempHomeVideoList.size() - bigPictureCount) % homeType.getColumns() != 0)
+ while (tempHomeVideoList != null && tempHomeVideoList.size() > 0 && (tempHomeVideoList.size() - bigPictureCount) % homeType.getColumns() != 0)
tempHomeVideoList.remove(tempHomeVideoList.size() - 1);
homeType.setHomeVideoList(tempHomeVideoList);
}
+
+ try {
+ if (VersionUtil.isGraterThan387(acceptData.getPlatform(), acceptData.getVersion()) && dataKey.equalsIgnoreCase("recommend")) {
+ DetailSystemConfig adContent = configService.getConfigByKey("home_banner_ad_content", acceptData.getDetailSystem(), acceptData.getVersion());
+ DetailSystemConfig adPosition = configService.getConfigByKey("home_banner_ad_position", acceptData.getDetailSystem(), acceptData.getVersion());
+ if (adContent != null && adPosition != null) {
+ HomeType.HomeTypeAd ad = new Gson().fromJson(adContent.getValue(), HomeType.HomeTypeAd.class);
+ int adP = Integer.parseInt(adPosition.getValue());
+ if (adP < homeTypeList.size() && adP > -1) {
+ HomeType homeType = new HomeType();
+ homeType.setAd(ad);
+ if ("1".equalsIgnoreCase(page)) {
+ homeTypeList.add(adP, homeType);
+ }
+ count++;
+ }
+ }
+ }
+ } catch (Exception e) {
+
+ }
+
JSONObject object = new JSONObject();
JSONArray array = new JSONArray();
@@ -370,12 +398,12 @@
return;
}
- List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
+ List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
//鏌ヨ澶у浘
List<HomeVideo> bigPictureVideos = new ArrayList<>();
if (homeType.isBigPicture()) {
- bigPictureVideos = homeTypeService.getHomeVideoList(homeTypeId, org.yeshi.utils.StringUtil.concat(resourceList, ","), resourceList, null, 1, 1);
+ bigPictureVideos = homeVideoService.getHomeVideoList(homeTypeId, org.yeshi.utils.StringUtil.concat(resourceList, ","), resourceList, null, 1, 1);
if (bigPictureVideos != null && bigPictureVideos.size() > 0)
bigPictureVideos.get(0).setBigPicture(true);
}
@@ -385,7 +413,7 @@
JSONObject object = new JSONObject();
String resourceKey = StringUtil.Md5(org.yeshi.utils.StringUtil.concat(resourceList, ","));
- Long count = homeTypeService.getHomeVideoListCount(homeTypeId, resourceKey, resourceList, null);
+ Long count = homeVideoService.getHomeVideoListCount(homeTypeId, resourceKey, resourceList, null);
if (count - bigPictureVideos.size() > homeType.getNumber()) {
while ((count - bigPictureVideos.size()) % homeType.getNumber() != 0) {
@@ -394,7 +422,7 @@
}
- List<HomeVideo> list1 = homeTypeService.getHomeVideoList(homeTypeId, resourceKey, resourceList, homeType.isBigPicture(), page, homeType.getNumber());
+ List<HomeVideo> list1 = homeVideoService.getHomeVideoList(homeTypeId, resourceKey, resourceList, homeType.isBigPicture(), page, homeType.getNumber());
List<HomeVideo> list = new ArrayList<>();
list.addAll(bigPictureVideos);
list.addAll(list1);
@@ -441,7 +469,7 @@
List<HomeVideo> homeVideoList = homeTypeService.getVideoByTypes(type, pageIndex, detailSystem);
- List<VideoInfo> list = new ArrayList<VideoInfo>();
+ List<VideoInfo> list = new ArrayList<>();
for (int i = 0; i < homeVideoList.size(); i++) {
HomeVideo hv = (HomeVideo) homeVideoList.get(i);
if (acceptData.getPackageName().contains("doudou")) {
@@ -493,6 +521,9 @@
@Resource
private TencentVideoUtil tencentVideoUtil;
+
+ @Resource
+ private BilibiliUtil bilibiliUtil;
@Resource
private VideoPlayStatisticManager videoPlayStatisticManager;
@@ -557,6 +588,14 @@
} else if (Integer.parseInt(resourceId) == TencentVideoUtil.RESOURCE_ID) {
try {
info = tencentVideoUtil.getVideoInfo(internetSearchVideo, pageSize);
+ } catch (Exception e) {
+ logger.error("鍏ㄧ綉鎼滆棰戣鎯呭嚭閿欙細" + videoId);
+ out.print(JsonUtil.loadFalseAdmin(e.getMessage()));
+ return;
+ }
+ } else if (Integer.parseInt(resourceId) == BilibiliUtil.RESOURCE_ID) {
+ try {
+ info = bilibiliUtil.getVideoInfo(internetSearchVideo, pageSize);
} catch (Exception e) {
logger.error("鍏ㄧ綉鎼滆棰戣鎯呭嚭閿欙細" + videoId);
out.print(JsonUtil.loadFalseAdmin(e.getMessage()));
@@ -633,7 +672,7 @@
DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());
- List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
+ List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
//鑾峰彇浼樺厛灞曠ず鐨勬笭閬�
if (StringUtil.isNullOrEmpty(resourceId)) {
@@ -708,10 +747,8 @@
ThreadUtil.run(new Runnable() {
@Override
public void run() {
- //姣忓ぉ灞曠ず涓�娆″箍鍛�
- long showTime = TimeUtil.convertGernalTime(TimeUtil.getGernalTime(System.currentTimeMillis() + 1000 * 60 * 60 * 24L, "yyyy-MM-dd"), "yyyy-MM-dd");
-
- deviceAdStrategyService.setVideoDetailAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), showTime, null);
+ //鐪婸P瑙嗛
+ deviceAdStrategyService.watchPPTVVideo(acceptData.getUtdId(), acceptData.getDetailSystem().getId());
}
});
return;
@@ -773,12 +810,20 @@
if (VersionUtil.isGraterThan390(acceptData.getPlatform(), acceptData.getVersion())) {
String sessionId = createVideoDetailSessionId(acceptData, loginUid, info.getName(), videoId, resourceId);
- out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(new VideoDetailVO(Utils.convertVideo(info), deviceAdStrategyService.getVideoDetailAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), loginUid), attention, sessionId)), array.toString(),
+ out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(new VideoDetailVO(Utils.convertVideo(info), deviceAdStrategyService.getVideoDetailAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), loginUid, videoId, from), attention, sessionId)), array.toString(),
obj.toString()));
} else {
out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(Utils.convertVideo(info)), array.toString(),
obj.toString()));
}
+
+ ThreadUtil.run(new Runnable() {
+ @Override
+ public void run() {
+ long showTime = TimeUtil.convertGernalTime(TimeUtil.getGernalTime(System.currentTimeMillis() + 1000 * 60 * 60 * 24L, "yyyy-MM-dd"), "yyyy-MM-dd");
+ deviceAdStrategyService.setShortVideoAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), videoId, from, showTime, null);
+ }
+ });
}
--
Gitblit v1.8.0