短视频爬虫，短视频搜索

admin

2021-07-30 a66b556036c2b3936a51fd7b7e54a204eb31dc14

 src/main/java/com/yeshi/buwan/controller/parser/HomeParser.java

@@ -18,15 +18,22 @@
import com.yeshi.buwan.domain.video.InternetSearchVideo;
import com.yeshi.buwan.domain.video.VideoWatchHistory;
import com.yeshi.buwan.dto.log.BaseLog;
import com.yeshi.buwan.dto.statistic.video.VideoDetailStatisticData;
import com.yeshi.buwan.mogotv.MogoTVUtil;
import com.yeshi.buwan.pptv.PPTVApiUtil;
import com.yeshi.buwan.pptv.PPTVUtil;
import com.yeshi.buwan.pptv.entity.PPTVProgram;
import com.yeshi.buwan.pptv.entity.PPTVSeries;
import com.yeshi.buwan.pptv.entity.VideoPPTVMap;
import com.yeshi.buwan.service.imp.*;
import com.yeshi.buwan.service.inter.ad.DeviceAdStrategyService;
import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService;
import com.yeshi.buwan.service.inter.juhe.PPTVService;
import com.yeshi.buwan.service.inter.juhe.YouKuService;
import com.yeshi.buwan.service.inter.recommend.HomeRecommendSpecialService;
import com.yeshi.buwan.service.inter.recommend.HomeVideoService;
import com.yeshi.buwan.service.inter.video.VideoWatchHistoryService;
import com.yeshi.buwan.service.manager.VideoPlayStatisticManager;
import com.yeshi.buwan.tencent.TencentVideoUtil;
import com.yeshi.buwan.util.*;
import com.yeshi.buwan.util.annotation.RequireUid;
@@ -34,6 +41,7 @@
import com.yeshi.buwan.util.log.LoggerUtil;
import com.yeshi.buwan.util.log.UserActiveLogFactory;
import com.yeshi.buwan.util.log.VideoLogFactory;
import com.yeshi.buwan.videos.bilibili.BilibiliUtil;
import com.yeshi.buwan.vo.AcceptData;
import com.yeshi.buwan.vo.video.VideoDetailVO;
import com.yeshi.buwan.youku.YouKuUtil;
@@ -65,6 +73,10 @@
    private HomeAdService homeAdService;
    @Resource
    private HomeTypeService homeTypeService;

    @Resource
    private HomeVideoService homeVideoService;

    @Resource
    private DetailSystemConfigService configService;
    @Resource
@@ -103,6 +115,9 @@

    @Resource
    private HomeRecommendSpecialService homeRecommendSpecialService;

    @Resource
    private DeviceAdStrategyService deviceAdStrategyService;

    private final Logger userPlayLogger = LoggerFactory.getLogger("videoPlayUser");

@@ -185,7 +200,7 @@
        DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());
        JSONObject object = new JSONObject();

        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
        List<HomeType> list1 = homeTypeService.getHomeType(detailSystem.getId(), acceptData.getPlatform(), acceptData.getVersion(), resourceList,
                CacheUtil.getMD5Long(resourceList), -1, special != null ? special.getDataKey() : null);
        List<HomeType> list = new ArrayList<>();
@@ -283,6 +298,76 @@
    }


    // 首页类型获取
    public void getHomeTypeNew(AcceptData acceptData, HttpServletRequest request, PrintWriter out) {
        // 20170914 一级视频分类ID
        String page = request.getParameter("Page");
        String pageSize = request.getParameter("PageSize");
        String dataKey = request.getParameter("DataKey");

        long count = homeTypeService.countHomeType(acceptData.getDetailSystem().getId(), acceptData.getVersion(), dataKey);

        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
        List<HomeType> homeTypeList = homeTypeService.getHomeTypeList(acceptData.getDetailSystem().getId(), acceptData.getVersion(), dataKey, Integer.parseInt(page), Integer.parseInt(pageSize));

        if (homeTypeList != null)
            for (HomeType homeType : homeTypeList) {
                String resourceKey = StringUtil.Md5(org.yeshi.utils.StringUtil.concat(resourceList, ","));

                long bigPictureCount = homeType.isBigPicture() ? 1 : 0;// homeTypeService.getHomeVideoListCount(homeType.getId(), resourceKey, resourceList, true);

                List<HomeVideo> homeVideoList = homeVideoService.getHomeVideoList(homeType.getId(), resourceKey, resourceList, null, 1, homeType.getNumber() + (int) bigPictureCount);
                if (bigPictureCount > 0 && homeVideoList.size() > 0) {
                    homeVideoList.get(0).setBigPicture(true);
                }
                long videoCount = homeVideoService.getHomeVideoListCount(homeType.getId(), resourceKey, resourceList, null);

                if (videoCount - bigPictureCount > homeType.getNumber()) {
                    while ((videoCount - bigPictureCount) % homeType.getNumber() != 0) {
                        videoCount--;
                    }
                }
                homeType.setCount((int) videoCount);
                List<HomeVideo> tempHomeVideoList = new ArrayList<>();
                tempHomeVideoList.addAll(homeVideoList);
                while (tempHomeVideoList != null && tempHomeVideoList.size() > 0 && (tempHomeVideoList.size() - bigPictureCount) % homeType.getColumns() != 0)
                    tempHomeVideoList.remove(tempHomeVideoList.size() - 1);

                homeType.setHomeVideoList(tempHomeVideoList);
            }

        try {
            if (VersionUtil.isGraterThan387(acceptData.getPlatform(), acceptData.getVersion()) && dataKey.equalsIgnoreCase("recommend")) {
                DetailSystemConfig adContent = configService.getConfigByKey("home_banner_ad_content", acceptData.getDetailSystem(), acceptData.getVersion());
                DetailSystemConfig adPosition = configService.getConfigByKey("home_banner_ad_position", acceptData.getDetailSystem(), acceptData.getVersion());
                if (adContent != null && adPosition != null) {
                    HomeType.HomeTypeAd ad = new Gson().fromJson(adContent.getValue(), HomeType.HomeTypeAd.class);
                    int adP = Integer.parseInt(adPosition.getValue());
                    if (adP < homeTypeList.size() && adP > -1) {
                        HomeType homeType = new HomeType();
                        homeType.setAd(ad);
                        if ("1".equalsIgnoreCase(page)) {
                            homeTypeList.add(adP, homeType);
                        }
                        count++;
                    }
                }
            }
        } catch (Exception e) {

        }


        JSONObject object = new JSONObject();
        JSONArray array = new JSONArray();
        for (int i = 0; i < homeTypeList.size(); i++)
            array.add(StringUtil.outPutResultJson(homeTypeList.get(i)));
        object.put("data", array);
        object.put("count", count + "");
        out.print(JsonUtil.loadTrueJson(object.toString()));
    }


    public HomeType convertHomeVideoList(HomeType type) {
        List<VideoInfo> vlist = new ArrayList<VideoInfo>();
        List<HomeVideo> list = type.getHomeVideoList();
@@ -313,22 +398,36 @@
            return;
        }

        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());
        //查询大图
        List<HomeVideo> bigPictureVideos = homeTypeService.getHomeVideoList(homeTypeId, resourceList, true, page, homeType.getNumber());
        List<HomeVideo> bigPictureVideos = new ArrayList<>();

        if (homeType.isBigPicture()) {
            bigPictureVideos = homeVideoService.getHomeVideoList(homeTypeId, org.yeshi.utils.StringUtil.concat(resourceList, ","), resourceList, null, 1, 1);
            if (bigPictureVideos != null && bigPictureVideos.size() > 0)
                bigPictureVideos.get(0).setBigPicture(true);
        }

        //大图需要每次都返回

        JSONObject object = new JSONObject();
        String resourceKey = StringUtil.Md5(org.yeshi.utils.StringUtil.concat(resourceList, ","));

        Long count = homeTypeService.getHomeVideoListCount(homeTypeId, resourceList, null);
        List<HomeVideo> list1 = homeTypeService.getHomeVideoList(homeTypeId, resourceList, false, page, homeType.getNumber());
        Long count = homeVideoService.getHomeVideoListCount(homeTypeId, resourceKey, resourceList, null);
        if (count - bigPictureVideos.size() > homeType.getNumber()) {

            while ((count - bigPictureVideos.size()) % homeType.getNumber() != 0) {
                count--;
            }
        }


        List<HomeVideo> list1 = homeVideoService.getHomeVideoList(homeTypeId, resourceKey, resourceList, homeType.isBigPicture(), page, homeType.getNumber());
        List<HomeVideo> list = new ArrayList<>();
        list.addAll(bigPictureVideos);
        list.addAll(list1);
        JSONObject data = new JSONObject();
        data.put("count", count);
        object.put("count", count);
        JSONArray array = new JSONArray();
        for (int i = 0; i < list.size(); i++)
            array.add(StringUtil.outPutResultJson(list.get(i)));
@@ -340,7 +439,7 @@
        if (page >= totalPage) {
            page = 0;
        }
        data.put("page", page);
        object.put("page", page);
        out.print(JsonUtil.loadTrueJson(object.toString()));
    }

@@ -370,7 +469,7 @@

        List<HomeVideo> homeVideoList = homeTypeService.getVideoByTypes(type, pageIndex, detailSystem);

        List<VideoInfo> list = new ArrayList<VideoInfo>();
        List<VideoInfo> list = new ArrayList<>();
        for (int i = 0; i < homeVideoList.size(); i++) {
            HomeVideo hv = (HomeVideo) homeVideoList.get(i);
            if (acceptData.getPackageName().contains("doudou")) {
@@ -423,19 +522,53 @@
    @Resource
    private TencentVideoUtil tencentVideoUtil;

    @Resource
    private BilibiliUtil bilibiliUtil;

    @Resource
    private VideoPlayStatisticManager videoPlayStatisticManager;


    private String createVideoDetailSessionId(AcceptData acceptData, String uid, String videoName, String videoId, String resourceId) {

        try {
            VideoDetailStatisticData data = new VideoDetailStatisticData();
            data.setDetailSystemId(acceptData.getDetailSystem().getId());
            data.setLoginUid(uid);
            data.setResourceId(Integer.parseInt(resourceId));
            data.setUtdId(acceptData.getUtdId());
            data.setVersion(acceptData.getVersion());
            data.setVideoId(videoId);
            data.setVideoName(videoName);

            return videoPlayStatisticManager.createVideoDetailSessionId(data);
        } catch (Exception e) {

        }
        return null;
    }


    public void getVideoDetailForInternetSearch(AcceptData acceptData, String videoId, HttpServletRequest request, PrintWriter out) {
        String loginUid = request.getParameter("LoginUid");
        String from = request.getParameter("From");
        InternetSearchVideo internetSearchVideo = internetSearchVideoService.selectByPrimaryKey(videoId);
        String resourceId = request.getParameter("ResourceId");
        String position = request.getParameter("Position");
        int pageSize = 100;
        if (!StringUtil.isNullOrEmpty(position)) {
            if (VersionUtil.isGraterThan391(acceptData.getPlatform(), acceptData.getVersion()))
                pageSize = (Integer.parseInt(position) / 100 + 1) * 100;
        }


        if (StringUtil.isNullOrEmpty(resourceId)) {
            resourceId = internetSearchVideo.getResourceIds().split(",")[0];
        }
        VideoInfo info = null;
        if (Integer.parseInt(resourceId) == YouKuUtil.RESOURCE_ID) {
            try {
                info = youKuUtil.getVideoInfo(internetSearchVideo);
                info = youKuUtil.getVideoInfo(internetSearchVideo, pageSize);

            } catch (Exception e) {
                logger.error("全网搜视频详情出错：" + videoId);
@@ -446,7 +579,7 @@

        } else if (Integer.parseInt(resourceId) == MogoTVUtil.RESOURCE_ID) {
            try {
                info = mogoTVUtil.getVideoInfo(internetSearchVideo);
                info = mogoTVUtil.getVideoInfo(internetSearchVideo, pageSize);
            } catch (Exception e) {
                logger.error("全网搜视频详情出错：" + videoId);
                out.print(JsonUtil.loadFalseAdmin(e.getMessage()));
@@ -454,13 +587,24 @@
            }
        } else if (Integer.parseInt(resourceId) == TencentVideoUtil.RESOURCE_ID) {
            try {
                info = tencentVideoUtil.getVideoInfo(internetSearchVideo);
                info = tencentVideoUtil.getVideoInfo(internetSearchVideo, pageSize);
            } catch (Exception e) {
                logger.error("全网搜视频详情出错：" + videoId);
                out.print(JsonUtil.loadFalseAdmin(e.getMessage()));
                return;
            }
        } else if (Integer.parseInt(resourceId) == BilibiliUtil.RESOURCE_ID) {
            try {
                info = bilibiliUtil.getVideoInfo(internetSearchVideo, pageSize);
            } catch (Exception e) {
                logger.error("全网搜视频详情出错：" + videoId);
                out.print(JsonUtil.loadFalseAdmin(e.getMessage()));
                return;
            }
        }

        if (info != null)
            info.setPageSize(pageSize);

        LoggerUtil.getUserActiveLogger().info(UserActiveLogFactory.createVideoDetail(new BaseLog(acceptData, loginUid), videoId, info.getName(), from));

@@ -470,7 +614,7 @@
        JSONObject obj = new JSONObject();

        if (VersionUtil.isGraterThan390(acceptData.getPlatform(), acceptData.getVersion())) {
            out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(new VideoDetailVO(Utils.convertVideo(info), new VideoDetailVO.VideoAdInfo(true, false), null)), array.toString(),
            out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(new VideoDetailVO(Utils.convertVideo(info), deviceAdStrategyService.getVideoDetailAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), loginUid), null, createVideoDetailSessionId(acceptData, loginUid, info.getName(), videoId, resourceId))), array.toString(),
                    obj.toString()));
        } else {
            out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(Utils.convertVideo(info)), array.toString(),
@@ -482,7 +626,32 @@
    public void getVideoDetail(AcceptData acceptData, HttpServletRequest request, PrintWriter out) {
        String videoId = request.getParameter("VideoId");
        if (!NumberUtil.isNumeric(videoId)) {
            getVideoDetailForInternetSearch(acceptData, videoId, request, out);
            String[] cidVid = PPTVUtil.parseCidAndVidFromVisualVideoId(videoId);
            if (cidVid == null) {
                getVideoDetailForInternetSearch(acceptData, videoId, request, out);
                return;
            }
            String cid = cidVid[0];
            String vid = cidVid[1];

            PPTVSeries series = pptvService.getSeriesDetail(cid);
            if (series == null) {
                series = PPTVApiUtil.getDetail(cid);
            }

            if (series != null) {
                JSONObject data = new JSONObject();
                String playUrl = PPTVUtil.getPlayUrl(series, pptvService.selectProgramById(vid));
                data.put("videoId", videoId);
                data.put("playUrl", playUrl);
                data.put("name", series.getName());
                data.put("pptv", true);
                data.put("vip", PPTVUtil.isVIPVideo(series.getFree()));
                out.print(JsonUtil.loadTrueJson(data.toString()));
            } else {
                out.print(JsonUtil.loadFalseJson("已下线"));
            }

            return;
        }

@@ -503,7 +672,7 @@

        DetailSystem detailSystem = systemService.getDetailSystemByPackage(acceptData.getPackageName());

        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion());
        List<Long> resourceList = videoResouceUtil.getAvailableResourceIds(acceptData.getDetailSystem(), acceptData.getVersion(),acceptData.getChannel());

        //获取优先展示的渠道
        if (StringUtil.isNullOrEmpty(resourceId)) {
@@ -538,7 +707,7 @@
        //PPTV网页播放
        if (resourceId != null && Integer.parseInt(resourceId) == PPTVUtil.RESOURCE_ID) {
            JSONObject data = new JSONObject();
            PPTVSeries series = pptvService.getSeriesDetailByVideoId(videoId);
            VideoPPTVMap map = pptvService.getVideoPPTVMapByVideoId(videoId);
            int p = 0;
            if (!StringUtil.isNullOrEmpty(position)) {
                p = Integer.parseInt(position);
@@ -550,15 +719,38 @@
                }
            }

            p = p >= series.getSeries().size() ? 0 : p;
            String videoName = "";
            String playUrl = null;

            String playUrl = PPTVUtil.getPlayUrl(series, series.getSeries().get(p));
            PPTVSeries series = pptvService.getSeriesDetail(map.getInfoId());

            if (map.getProgramCode() != null) {
                PPTVProgram program = pptvService.selectProgramById(map.getProgramCode());
                videoName = program.getProgramSetTiltle();
                playUrl = PPTVUtil.getPlayUrl(series, program);
            } else {
                videoName = series.getName();
                p = p >= series.getSeries().size() ? 0 : p;
                playUrl = PPTVUtil.getPlayUrl(series, series.getSeries().get(p));
            }

            String sessionId = createVideoDetailSessionId(acceptData, loginUid, videoName, videoId, resourceId);
            data.put("videoId", videoId);
            data.put("playUrl", playUrl);
            data.put("name", series.getName());
            data.put("name", videoName);
            data.put("pptv", true);
            data.put("vip", "1".equalsIgnoreCase(series.getFree()));
            data.put("sessionId", sessionId);
            out.print(JsonUtil.loadTrueJson(data.toString()));
            LoggerUtil.getUserActiveLogger().info(UserActiveLogFactory.createVideoDetail(new BaseLog(acceptData, loginUid), videoId, series.getName(), from));

            ThreadUtil.run(new Runnable() {
                @Override
                public void run() {
                    //看PP视频
                    deviceAdStrategyService.watchPPTVVideo(acceptData.getUtdId(), acceptData.getDetailSystem().getId());
                }
            });
            return;
        }

@@ -616,12 +808,22 @@
            }
        UserInfo user = userService.getUserInfo(acceptData.getUid());
        if (VersionUtil.isGraterThan390(acceptData.getPlatform(), acceptData.getVersion())) {
            out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(new VideoDetailVO(Utils.convertVideo(info), new VideoDetailVO.VideoAdInfo(true, true), attention)), array.toString(),
            String sessionId = createVideoDetailSessionId(acceptData, loginUid, info.getName(), videoId, resourceId);

            out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(new VideoDetailVO(Utils.convertVideo(info), deviceAdStrategyService.getVideoDetailAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), loginUid, videoId, from), attention, sessionId)), array.toString(),
                    obj.toString()));
        } else {
            out.print(JsonUtil.loadTrueJson(StringUtil.outPutResultJson(Utils.convertVideo(info)), array.toString(),
                    obj.toString()));
        }

        ThreadUtil.run(new Runnable() {
            @Override
            public void run() {
                long showTime = TimeUtil.convertGernalTime(TimeUtil.getGernalTime(System.currentTimeMillis() + 1000 * 60 * 60 * 24L, "yyyy-MM-dd"), "yyyy-MM-dd");
                deviceAdStrategyService.setShortVideoAdStrategy(acceptData.getUtdId(), acceptData.getDetailSystem().getId(), videoId, from, showTime, null);
            }
        });
    }


@@ -637,6 +839,7 @@
        String videoId = request.getParameter("VideoId");
        String resourceId = request.getParameter("ResourceId");
        String page = request.getParameter("Page");//页码
        String pageSize = request.getParameter("PageSize");//页

        if (StringUtil.isNullOrEmpty(videoId)) {
            out.print(JsonUtil.loadFalseJson("请上传VideoId"));
@@ -651,9 +854,13 @@
            out.print(JsonUtil.loadFalseJson("请上传Page"));
            return;
        }
        int pagesize = 100;
        if (!StringUtil.isNullOrEmpty(pageSize)) {
            pagesize = Integer.parseInt(pageSize);
        }


        List<VideoDetailInfo> list = videoDetailUtil.getVideoDetailList(videoId, new VideoResource(resourceId), Integer.parseInt(page), 100);
        List<VideoDetailInfo> list = videoDetailUtil.getVideoDetailList(videoId, new VideoResource(resourceId), Integer.parseInt(page), pagesize);
        if (list == null)
            list = new ArrayList<>();
        JSONObject data = new JSONObject();