From a66b556036c2b3936a51fd7b7e54a204eb31dc14 Mon Sep 17 00:00:00 2001
From: admin <weikou2014>
Date: 星期五, 30 七月 2021 19:13:14 +0800
Subject: [PATCH] 短视频爬虫，短视频搜索
---
src/main/java/com/yeshi/buwan/controller/parser/ClassParser.java | 333 ++++++++++++++++++++++++++++++++++++++++++++-----------
1 files changed, 264 insertions(+), 69 deletions(-)
diff --git a/src/main/java/com/yeshi/buwan/controller/parser/ClassParser.java b/src/main/java/com/yeshi/buwan/controller/parser/ClassParser.java
index d905c48..27e55e4 100644
--- a/src/main/java/com/yeshi/buwan/controller/parser/ClassParser.java
+++ b/src/main/java/com/yeshi/buwan/controller/parser/ClassParser.java
@@ -4,13 +4,22 @@
import com.yeshi.buwan.domain.ad.CommonAd;
import com.yeshi.buwan.domain.ad.CommonAdPositionAd;
import com.yeshi.buwan.domain.recommend.CategoryRecommendVideo;
+import com.yeshi.buwan.domain.recommend.HomeRecommendSpecial;
import com.yeshi.buwan.domain.special.Special;
+import com.yeshi.buwan.domain.system.DetailSystem;
+import com.yeshi.buwan.domain.system.DetailSystemConfig;
+import com.yeshi.buwan.dto.search.SolrResultDTO;
+import com.yeshi.buwan.dto.search.SolrVideoSearchFilter;
+import com.yeshi.buwan.pptv.PPTVUtil;
import com.yeshi.buwan.service.imp.*;
import com.yeshi.buwan.service.imp.recommend.CategoryRecommendVideoService;
+import com.yeshi.buwan.service.inter.recommend.HomeRecommendSpecialService;
+import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager;
import com.yeshi.buwan.util.*;
import com.yeshi.buwan.util.JuHe.VideoResourceUtil;
import com.yeshi.buwan.util.ad.CommonAdUtil;
import com.yeshi.buwan.util.annotation.RequireUid;
+import com.yeshi.buwan.util.video.VideoConstant;
import com.yeshi.buwan.vo.AcceptData;
import com.yeshi.buwan.vo.HomeClassVO;
import com.yeshi.buwan.vo.video.VideoListResultVO;
@@ -22,6 +31,7 @@
import javax.servlet.http.HttpServletRequest;
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
@Controller
@@ -55,6 +65,17 @@
@Resource
private RedisManager redisManager;
+ @Resource
+ private SolrAlbumVideoDataManager solrAlbumVideoDataManager;
+
+ @Resource
+ private SearchService searchService;
+
+ @Resource
+ private HomeRecommendSpecialService homeRecommendSpecialService;
+
+ @Resource
+ private DetailSystemConfigService detailSystemConfigService;
@RequireUid
public void getClass(AcceptData acceptData, HttpServletRequest request, PrintWriter out) {
@@ -102,8 +123,14 @@
if (!StringUtil.isNullOrEmpty(list.get(i).getPicture())) {
list.get(i).getType().setIcon(list.get(i).getPicture());
}
- typeList.add(list.get(i).getType());
+ if ("ios".equalsIgnoreCase(acceptData.getPlatform()) && acceptData.getVersion() >= Constant.IOS_ONLINING_VERSION) {
+ if (VideoConstant.isMainCategory(list.get(i).getType().getId())) {
+ typeList.add(list.get(i).getType());
+ }
+ } else
+ typeList.add(list.get(i).getType());
}
+
JSONObject object = new JSONObject();
object.put("count", (new StringBuilder(String.valueOf(typeList.size()))).toString());
@@ -138,26 +165,17 @@
}
}
-// if (acceptData.getPlatform().equalsIgnoreCase("android")) {
-// for (SuperVideoType type : list) {
-// if (type.getType().getName().contains("璧勮")) {
-// type.getType().setName("涓绘挱");
-// type.getType().setId(1111);
-// type.getType().setIcon("http://img.zcool.cn/community/017fdb57610d8a0000012e7e74b496.png");
-// break;
-// }
-// }
-// }
- if ("android".equalsIgnoreCase(acceptData.getPlatform()) && acceptData.getVersion() > 53 && !"qq".equalsIgnoreCase(acceptData.getChannel())) {
+ //Android骞冲彴3.8.7涔嬪悗杩斿洖VIP
+ if ("android".equalsIgnoreCase(acceptData.getPlatform()) && acceptData.getVersion() >= 105) {
+
SuperVideoType sty = new SuperVideoType();
sty.setCreatetime(0 + "");
sty.setPicture("");
- VideoType vt = new VideoType(22222);
- vt.setName("缇庡コ鐩存挱");
+ VideoType vt = new VideoType(Constant.VIDEO_TYPE_VIP);
+ vt.setName("VIP");
vt.setShow("1");
- vt.setCategoryType("http://m.v.6.cn/event/porkvideo?src=9n8wc5medm&nologo=1&t=2");
- vt.setIcon("https://hbimg.huabanimg.com/12834704bb4aa39342c2fb51e0c644181b13997b70eb-CqlE1I_fw658/format/webp");
+ vt.setIcon("https://hbimg.huabanimg.com/4690ea8f8144f3d46c11e417c77daa5debcb71f9201f-WpAbfw_fw658/format/jpg");
sty.setType(vt);
boolean isC = false;
for (SuperVideoType svt : list) {
@@ -166,10 +184,9 @@
break;
}
}
-
- if (!isC)
- list.add(sty);
+ list.add(sty);
}
+
List<VideoType> typelist = new ArrayList<VideoType>();
for (SuperVideoType sv : list) {
@@ -194,7 +211,7 @@
/**
* 鑾峰彇褰撳墠绯荤粺鎵�鏈夊悗鍙拌棰戝垎绫�
*
- * @param uid
+ * @param acceptData
* @param request
* @param out
*/
@@ -226,6 +243,44 @@
// continue;
// }
typelist.add(type);
+ }
+
+ if ("ios".equalsIgnoreCase(acceptData.getPlatform()) && acceptData.getVersion() >= Constant.IOS_ONLINING_VERSION) {
+ typelist = new ArrayList<>();
+ HomeClassVO vo = new HomeClassVO();
+ vo.setId(309L);
+ vo.setName("鎺ㄨ崘");
+ vo.setShowTitle("鎺ㄨ崘");
+ vo.setDataType(HomeClassVO.DATA_TYPE_RECOMMEND);
+ typelist.add(vo);
+
+ vo = new HomeClassVO();
+ vo.setId(VideoConstant.VIDEO_CATEGORY_DIANYING);
+ vo.setName("鐢靛奖");
+ vo.setShowTitle("鐢靛奖");
+ vo.setDataType(HomeClassVO.DATA_TYPE_CLASS);
+ typelist.add(vo);
+
+ vo = new HomeClassVO();
+ vo.setId(VideoConstant.VIDEO_CATEGORY_DIANSHIJU);
+ vo.setName("鐢佃鍓�");
+ vo.setShowTitle("鐢佃鍓�");
+ vo.setDataType(HomeClassVO.DATA_TYPE_CLASS);
+ typelist.add(vo);
+
+ vo = new HomeClassVO();
+ vo.setId(VideoConstant.VIDEO_CATEGORY_DONGMAN);
+ vo.setName("鍔ㄦ极");
+ vo.setShowTitle("鍔ㄦ极");
+ vo.setDataType(HomeClassVO.DATA_TYPE_CLASS);
+ typelist.add(vo);
+
+ vo = new HomeClassVO();
+ vo.setId(VideoConstant.VIDEO_CATEGORY_ZONGYI);
+ vo.setName("缁艰壓");
+ vo.setShowTitle("缁艰壓");
+ vo.setDataType(HomeClassVO.DATA_TYPE_CLASS);
+ typelist.add(vo);
}
JSONObject object = new JSONObject();
@@ -269,15 +324,33 @@
vo.setShowTitle(type.getShowTitle());
vo.setType(type.getType());
vo.setCategoryType(type.getCategoryType());
-
- typelist.add(vo);
+ //鍒犻櫎鑰佺増鏈腑鐨勯櫎寮�鍒嗙被鐨勯《閮ㄦ爣绛�
+ if (sv.getType().getId() != 309L && sv.getType().getId() != 310L)
+ typelist.add(vo);
}
- HomeClassVO vo = new HomeClassVO();
+ //IOS瀹℃牳鐗堟湰
+ DetailSystemConfig onLineVersion = detailSystemConfigService.getConfigByKey("ios_online_version", detailSystem, acceptData.getVersion());
+ if (onLineVersion != null && Integer.parseInt(onLineVersion.getValue()) <= acceptData.getVersion()) {
+ typelist.clear();
+ }
- vo.setDataType(HomeClassVO.DATA_TYPE_NOVEL);
- vo.setName("灏忚");
- typelist.add(1, vo);
+ //鑾峰彇棣栭〉鎺ㄨ崘鏍囩
+ List<HomeRecommendSpecial> specialList = homeRecommendSpecialService.listSpecialByDetailSystemId(acceptData.getDetailSystem().getId(), null, 1, Constant.pageCount);
+ if (specialList != null) {
+ for (int i = specialList.size() - 1; i >= 0; i--) {
+ HomeRecommendSpecial special = specialList.get(i);
+ HomeClassVO vo = new HomeClassVO();
+ vo.setId(Long.parseLong(special.getId()));
+ vo.setName(special.getName());
+ vo.setShowTitle(special.getName());
+ vo.setDataKey(special.getDataKey());
+ vo.setDataType(HomeClassVO.DATA_TYPE_RECOMMEND);
+ if (vo.getName().contains("楂樻竻"))
+ vo.setColor("#ff6600");
+ typelist.add(0, vo);
+ }
+ }
JSONObject object = new JSONObject();
object.put("count", (new StringBuilder(String.valueOf(typelist.size()))).toString());
@@ -286,7 +359,6 @@
array.add(StringUtil.outPutResultJson(typelist.get(j)));
object.put("data", array);
out.print(JsonUtil.loadTrueJson(object.toString()));
-
}
@@ -345,9 +417,9 @@
return;
}
- List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getPlatform(), acceptData.getVersion());
-
DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());
+
+ List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(detailSystem, acceptData.getVersion(), acceptData.getChannel());
List<VideoInfo> list = starService.getStarVideo(detailSystem.getId(), starId, pageIndex,
CacheUtil.getMD5Long(resourceList));
@@ -404,11 +476,12 @@
return;
}
- List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getPlatform(), acceptData.getVersion());
- DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());
+ DetailSystem detailSystem = acceptData.getDetailSystem();
UserInfo user = userService.getUserInfo(acceptData.getUid());
if (Utils.isTest(request, user, detailSystem.getId()))
detailSystem = systemService.getDetailSystemById(40 + "");
+
+ List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(), acceptData.getChannel());
String resourceMD5 = CacheUtil.getMD5Long(resourceList);
@@ -439,7 +512,7 @@
} else if (!StringUtil.isNullOrEmpty(homeType)) {
- list = new ArrayList<VideoInfo>();
+ list = new ArrayList<>();
List<HomeVideo> homeVideoList = homeTypeService.getVideoByTypes(homeType, pageIndex, detailSystem);
for (HomeVideo hv : homeVideoList) {
@@ -449,15 +522,17 @@
}
list.add(hv.getVideo());
}
+
count = (new StringBuilder(String.valueOf(homeTypeService.getVideoCountByTypes(homeType, pageIndex))))
.toString();
} else if (!StringUtil.isNullOrEmpty(videoType)) {
if (StringUtil.isNullOrEmpty(categoryType) || "genre".equalsIgnoreCase(categoryType)) {// 鎸夊垎绫绘绱㈡暟鎹�
+
if (!StringUtil.isNullOrEmpty(order)) {
Long startTime = System.currentTimeMillis();
List<VideoInfo> list1 = classService.getTypeVideoList(videoType, detailSystem, pageIndex, 20,
Integer.parseInt(order), resourceList, CacheUtil.getMD5Long(resourceList));
- list = new ArrayList<VideoInfo>();
+ list = new ArrayList<>();
if (list1 != null)
for (VideoInfo info : list1)
list.add(info);
@@ -466,25 +541,62 @@
} else {
List<VideoInfo> list1 = classService.getTypeVideoList(videoType, detailSystem, pageIndex, 20, 1,
resourceList, CacheUtil.getMD5Long(resourceList));
- list = new ArrayList<VideoInfo>();
+ list = new ArrayList<>();
if (list1 != null)
for (VideoInfo info : list1)
list.add(info);
}
count = 1200 + "";
+
+
} else {// 鎸夊湴鍖烘绱㈡暟鎹�
- String areaId = videoType;// 鍥界睄ID
- CategoryContry cc = categoryAreaService.getCategoryArea(areaId);
- String country = cc.getName();
- long typeid = cc.getParent().getCid();
- if (StringUtil.isNullOrEmpty(order))
- order = 1 + "";
- list = categoryAreaService.getVideoInfoByArea(typeid + "", detailSystem, pageIndex, 20,
- Integer.parseInt(order), resourceList, country,
- StringUtil.Md5(typeid + "-" + detailSystem + "-" + pageIndex + "-" + order + "-" + country
- + "-" + CacheUtil.getMD5Long(resourceList)));
- System.out.println("鍒楄〃鏈�缁堟暟閲�:" + list.size());
- count = 1000 + "";
+ //闈濾IP鍒嗙被
+ List<Integer> vipTypes = Arrays.asList(Constant.vipTypes);
+ if (vipTypes.contains(Integer.parseInt(videoType))) {
+ //VIP鍒嗙被
+ SolrVideoSearchFilter filter = new SolrVideoSearchFilter();
+ filter.setFreeType(1);
+ switch (Integer.parseInt(videoType)) {
+ case Constant.VIDEO_TYPE_VIP_MOVIE:
+ filter.setVideoType(VideoConstant.VIDEO_CATEGORY_DIANYING);
+ break;
+ case Constant.VIDEO_TYPE_VIP_TV:
+ filter.setVideoType(VideoConstant.VIDEO_CATEGORY_DIANSHIJU);
+ break;
+ case Constant.VIDEO_TYPE_VIP_CARTOON:
+ filter.setVideoType(VideoConstant.VIDEO_CATEGORY_DONGMAN);
+ break;
+ case Constant.VIDEO_TYPE_VIP_SHOW:
+ filter.setVideoType(VideoConstant.VIDEO_CATEGORY_ZONGYI);
+ break;
+ default:
+ }
+ filter.setResourceIds(new String[]{PPTVUtil.RESOURCE_ID + ""});
+
+ //鏇存柊鏃堕棿
+ if ("1".equalsIgnoreCase(order)) {
+ filter.setSortKey("updateTime");
+ } else {
+ //瑙傜湅娆℃暟
+ filter.setSortKey("watchcount");
+ }
+ SolrResultDTO resultDTO = solrAlbumVideoDataManager.find(filter, pageIndex, 20);
+ count = resultDTO.getTotalCount() + "";
+ list = searchService.convertSolrAlbumResultToVideo(resultDTO.getVideoList(), resourceList);
+ } else {
+ String areaId = videoType;// 鍥界睄ID
+ CategoryContry cc = categoryAreaService.getCategoryArea(areaId);
+ String country = cc.getName();
+ long typeid = cc.getParent().getCid();
+ if (StringUtil.isNullOrEmpty(order))
+ order = 1 + "";
+ list = categoryAreaService.getVideoInfoByArea(typeid + "", detailSystem, pageIndex, 20,
+ Integer.parseInt(order), resourceList, country,
+ StringUtil.Md5(typeid + "-" + detailSystem + "-" + pageIndex + "-" + order + "-" + country
+ + "-" + CacheUtil.getMD5Long(resourceList)));
+ System.out.println("鍒楄〃鏈�缁堟暟閲�:" + list.size());
+ count = 1000 + "";
+ }
}
}
list = banQuanService.getBanQuanVideo(list, detailSystem.getId(), CacheUtil.getMD5VideoInfo(list));
@@ -556,6 +668,14 @@
String parentId = request.getParameter("ParentId");
+ //鏄惁鍖呭惈VIP鐨勫垎绫�
+ boolean containsVIPType = false;
+ try {
+ List<Integer> vipTypeList = Arrays.asList(Constant.vipTypes);
+ containsVIPType = vipTypeList.contains(Integer.parseInt(parentId));
+ } catch (Exception e) {
+
+ }
if (StringUtil.isNullOrEmpty(parentId)) {
out.print(JsonUtil.loadFalseJson("璇蜂笂浼燩arentId"));
return;
@@ -611,9 +731,58 @@
list.add(vt);
}
+ if (acceptData.getVersion() > 60) { // 20170915鍒犻櫎鎺ㄨ崘
+ list.remove(0);
+ }
+
+ } else if (containsVIPType) {
+ //VIP鍒嗙被
+
+ //VIP鏍瑰垎绫�
+ if (Constant.VIDEO_TYPE_VIP == Integer.parseInt(parentId)) {
+ VideoType type = new VideoType(Constant.VIDEO_TYPE_VIP);
+ type.setName("鍏ㄩ儴");
+ type.setShow("1");
+ type.setCategoryType("area");
+ list.add(type);
+
+
+ type = new VideoType(Constant.VIDEO_TYPE_VIP_MOVIE);
+ type.setName("鐢靛奖");
+ type.setShow("1");
+ type.setCategoryType("area");
+ list.add(type);
+
+ type = new VideoType(Constant.VIDEO_TYPE_VIP_TV);
+ type.setName("鐢佃鍓�");
+ type.setShow("1");
+ type.setCategoryType("area");
+ list.add(type);
+
+
+ type = new VideoType(Constant.VIDEO_TYPE_VIP_CARTOON);
+ type.setName("鍔ㄦ极");
+ type.setShow("1");
+ type.setCategoryType("area");
+ list.add(type);
+
+// type = new VideoType(Constant.VIDEO_TYPE_VIP_SHOW);
+// type.setName("缁艰壓");
+// type.setShow("1");
+// type.setCategoryType("area");
+// list.add(type);
+ } else {
+ VideoType type = new VideoType(Integer.parseInt(parentId));
+ type.setName("鍏ㄩ儴");
+ type.setShow("1");
+ type.setCategoryType("area");
+ list.add(type);
+ }
+
+
} else {
List<VideoType> clist = classService.getFirstTypeList(parentId);
- list = new ArrayList<VideoType>();
+ list = new ArrayList<>();
if (clist != null)
for (VideoType vt : clist)
list.add(vt);
@@ -626,13 +795,14 @@
type.setShow("1");
type.setCategoryType("genre");
list.add(0, type);
+ if (acceptData.getVersion() > 60) { // 20170915鍒犻櫎鎺ㄨ崘
+ list.remove(0);
+ }
}
for (int i = 0; i < list.size(); i++) {
array.add(StringUtil.outPutResultJson(list.get(i)));
}
- if (acceptData.getVersion() > 60) { // 20170915鍒犻櫎鎺ㄨ崘
- array.remove(0);
- }
+
object.put("count", array.size());
object.put("data", array);
out.print(JsonUtil.loadTrueJson(object.toString()));
@@ -681,11 +851,33 @@
out.print(JsonUtil.loadTrueJson(object.toString()));
}
+ @Resource
+ private VideoResourceService videoResourceService;
+
@RequireUid
public void getSpecialDetail(AcceptData acceptData, HttpServletRequest request, PrintWriter out) {
String specialid = request.getParameter("Id");
Special special = specialService.getSpecial(specialid);
List<VideoInfo> list = specialService.getSpecialVideoList(specialid);
+ List<Long> availableResourceIds = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(), acceptData.getChannel());
+ List<VideoInfo> tempList = new ArrayList<>();
+ for (VideoInfo videoInfo : list) {
+ List<VideoResource> resourceList = videoResourceService.getResourceByVideo(videoInfo);
+ List<Long> rids = new ArrayList<>();
+ for (VideoResource vr : resourceList) {
+ if (availableResourceIds.contains(Long.parseLong(vr.getId()))) {
+ rids.add(Long.parseLong(vr.getId()));
+ break;
+ }
+ }
+ if (rids.size() > 0) {
+ tempList.add(videoInfo);
+ }
+ }
+
+ list = tempList;
+
+
JSONObject object = new JSONObject();
object.put("count", list.size());
JSONArray array = new JSONArray();
@@ -745,6 +937,8 @@
if ("310".equals(typeid)) { // 310鍋氱殑鐗规畩澶勭悊:鍓嶇澶勭悊鐨勬槸HomeAd锛屾墍鏈夐渶瑕佽繑鍥瀐omeAD鐨勫舰寮�
Object vi = json.get("VideoInfo");
json.put("Video", vi);
+ //璁剧疆banner鐨勬瘮渚�
+ object.put("bannerSizeRate", "0.56");
}
json.put("LinkType", "1");
array.add(json);
@@ -756,26 +950,27 @@
public void getRecommendCategoryVideoList(AcceptData acceptData, HttpServletRequest request, PrintWriter out) {
String typeid = request.getParameter("Type");
DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());
- // int count =
- // categoryRecommendVideoService.getRankVideoNumber(Integer.parseInt(typeid));
- List<VideoInfo> rankList = categoryRecommendCacheVideoService.getVideoListByRank(Integer.parseInt(typeid),
- detailSystem.getId());
- List<VideoInfo> list = new ArrayList<VideoInfo>();
+ List<VideoInfo> rankList = null;
+// if ("ios".equalsIgnoreCase(acceptData.getPlatform()) && acceptData.getVersion() >= Constant.IOS_ONLINING_VERSION) {
+// SolrVideoSearchFilter filter = new SolrVideoSearchFilter();
+// filter.setResourceIds(new String[]{25 + ""});
+// filter.setVideoType(Integer.parseInt(typeid));
+// filter.setFreeType(0);
+// SolrResultDTO dto = solrAlbumVideoDataManager.find(filter, 1, 30);
+// List<SolrAlbumVideo> list = dto.getVideoList();
+// rankList = new ArrayList<>();
+// for (SolrAlbumVideo video : list) {
+// rankList.add(VideoInfoFactory.create(video));
+// }
+// }
+// else {
+ rankList = categoryRecommendCacheVideoService.getVideoListByRank(Integer.parseInt(typeid),
+ detailSystem.getId(), acceptData.getPlatform(), acceptData.getVersion(), acceptData.getChannel());
+// }
+
+
+ List<VideoInfo> list = new ArrayList<>();
list.addAll(rankList);
- // List<VideoInfo> list1 =
- // recommendService.getCategoryRecommendVideoList(Integer.parseInt(typeid),
- // count);
- // int crvs = list1.size();
- // for (CategoryRecommendCacheVideo crcv : rankList) {
- // int rank = crcv.getRank();
- // if(crvs > 0 && rank > 0 && rank <= crvs ){
- // list1.add(rank-1, crcv.getVideoInfo());
- // }
- // }
- // list.addAll(list1.subList(0, list1.size()>count?count:list1.size()));
- //
- // list = banQuanService.getBanQuanVideo(list, detailSystem.getId(),
- // CacheUtil.getMD5VideoInfo(list));
for (VideoInfo vi : list) {// 娓呴櫎鏃犵敤鏁版嵁,闃叉娴垂甯﹀,鏇存敼鍥剧墖
vi.setIntroduction("");
vi.setMainActor("");
--
Gitblit v1.8.0