风行网页版爬虫

admin

2024-09-05 ab35ac8b769b2d9816dffb33a64f2c6f7bd5dd6e

 src/main/java/com/yeshi/buwan/service/imp/SearchService.java

@@ -10,10 +10,10 @@
import com.yeshi.buwan.domain.web.HotSearchAdmin;
import com.yeshi.buwan.dto.search.SolrResultDTO;
import com.yeshi.buwan.dto.search.SolrVideoSearchFilter;
import com.yeshi.buwan.pptv.PPTVUtil;
import com.yeshi.buwan.videos.pptv.PPTVUtil;
import com.yeshi.buwan.service.inter.video.VideoInfoExtraService;
import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager;
import com.yeshi.buwan.service.manager.SolrInternetSearchVideoDataManager;
import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager;
import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager;
import com.yeshi.buwan.util.*;
import com.yeshi.buwan.util.factory.VideoInfoFactory;
import com.yeshi.buwan.util.video.VideoConstant;
@@ -34,6 +34,9 @@
@Service
public class SearchService {
    private Logger logger = LoggerFactory.getLogger(SearchService.class);
    private Logger debugLogger = LoggerFactory.getLogger("debug");

    private final String ALBUM_SEARCH_FUZZY_KEYS = "《,》,。";


    @Resource
@@ -288,18 +291,25 @@
    }


    public SearchResult searchAlbum(int videoType, String key, int page, int pageSize) {
    public SearchResult searchAlbum(int videoType, String key, List<Long> resourceIds, int page, int pageSize, boolean fuzzy) {
        List<VideoInfo> localList = new ArrayList<>();
        //专辑视频集合
        Set<String> solrAlbumVids = new HashSet<>();

        SolrVideoSearchFilter filter = new SolrVideoSearchFilter();
        filter.setFuzzy(fuzzy);
        filter.setKey(key);
        if (videoType == Constant.SEARCH_RESULT_TYPE_HIGH_DEFINITION) {
            filter.setResourceIds(new String[]{PPTVUtil.RESOURCE_ID + ""});
            filter.setResourceIds(Arrays.asList(new String[]{PPTVUtil.RESOURCE_ID + ""}));
            logger.info("高清搜索# key：{}", key);
        } else
        } else {
            List<String> rids = new ArrayList<>();
            for (Long rid : resourceIds) {
                rids.add(rid + "");
            }
            filter.setResourceIds(rids);
            filter.setVideoType(videoType == 0 ? null : videoType);
        }
        filter.setContentType(1);

        SolrResultDTO solrResultDTO = solrAlbumDataManager.find(filter, page, pageSize);
@@ -309,14 +319,14 @@
            albumCount = solrResultDTO.getTotalCount();
            for (SolrAlbumVideo sv : (List<SolrAlbumVideo>) solrResultDTO.getVideoList()) {
                VideoInfo video = VideoInfoFactory.create(sv);
                albumSet.add(video.getName() + "#" + video.getVideoType().getId());
                albumSet.add(video.getName() + "#" + video.getVideoType().getId() + "#" + video.getYear());
                video = loadAlbumData(video);
                localList.add(video);
                solrAlbumVids.add(video.getId());
            }
        }

        //第一页,不为高清,主搜索引擎无法搜索到内容时搜索
        //第一页,不为高清时搜索
        if (page == 1 && videoType != Constant.SEARCH_RESULT_TYPE_HIGH_DEFINITION) {
            //page为1时 全网搜
            SolrResultDTO internetSearchResultDTO = solrInternetSearchVideoDataManager.find(filter, 1, 20);
@@ -327,34 +337,51 @@
                    isvList.addAll(iList);
                //删除前面有的数据，根据rootType与名称筛选
                for (int i = 0; i < isvList.size(); i++) {
                    String uniqueId = isvList.get(i).getName() + "#" + isvList.get(i).getRootType();
                    String uniqueId = isvList.get(i).getName() + "#" + isvList.get(i).getRootType() + "#" + isvList.get(i).getYear();
                    if (albumSet.contains(uniqueId)) {
                        isvList.remove(i);
                        i--;
                    }
                }
                for (InternetSearchVideo sv : isvList) {
                    VideoInfo video = VideoInfoFactory.create(sv);
                    video = loadAlbumData(video);
                    if (albumCount < 3) {
                        localList.add(video);
                    } else {
                        if (video.getName().equalsIgnoreCase(key))

                //精确匹配，不分词
                if (!fuzzy) {
                    for (InternetSearchVideo sv : isvList) {
                        VideoInfo video = VideoInfoFactory.create(sv);
                        video = loadAlbumData(video);
                        //第一个专辑数量
                        if (albumCount < 3) {
                            localList.add(video);
                        } else {
                            if (video.getName().equalsIgnoreCase(key))
                                localList.add(video);
                        }
                    }
                } else {
                    for (InternetSearchVideo sv : isvList) {
                        VideoInfo video = VideoInfoFactory.create(sv);
                        video = loadAlbumData(video);
                        localList.add(video);
                    }
                }
            }
            //按关键词的匹配度排序
            Comparator<VideoInfo> cm = new Comparator<VideoInfo>() {
                @Override
                public int compare(VideoInfo o1, VideoInfo o2) {
            Comparator<VideoInfo> cm = (VideoInfo o1, VideoInfo o2) -> {
                if (o1.getDefinition() - o2.getDefinition() == 0) {
                    float s1 = CompareStrSimUtil.getSimilarityRatio(o1.getName(), key, true);
                    float s2 = CompareStrSimUtil.getSimilarityRatio(o2.getName(), key, true);
                    if (s1 != s2)//相似度越高越靠前
//                        if (s1 != s2)//相似度越高越靠前
                    try {
                        return s1 != s2 ? (s2 > s1 ? 1 : -1) : (Integer.parseInt(o2.getYear()) - Integer.parseInt(o1.getYear()));
                    } catch (Exception e) {
                        return s2 > s1 ? 1 : -1;
                    else
                        return o1.getDefinition() - o2.getDefinition();//高清排在前
                    }
//                        else
//                            return o1.getDefinition() - o2.getDefinition();//高清排在前
                } else {
                    return o2.getDefinition() - o1.getDefinition();//高清排在前
                }

            };

            Collections.sort(localList, cm);
@@ -416,25 +443,35 @@
        if (!StringUtil.isNullOrEmpty(key) && key.length() > 30)
            key = key.substring(0, 30);
        sh.setKey(key);
        UserInfo user = new UserInfo();
        user.setId(uid);
        sh.setUser(user);
        if (uid != null) {
            UserInfo user = new UserInfo();
            user.setId(uid);
            sh.setUser(user);
        }

        searchDao.create(sh);
//        searchDao.create(sh);
        // 查询数据库

        //专辑数量
        long albumCount = 0L;
        List<VideoInfo> list = new ArrayList<>();
        List<VideoInfo> localList = null;
        long startt = System.currentTimeMillis();
        try {
            long startt = System.currentTimeMillis();

            localList = null;

            localList = new ArrayList<>();
            //先搜索专辑
            int pageSize = 20;
            SearchResult searchResult = searchAlbum(videoType, key, page, pageSize);
            SearchResult searchResult = searchAlbum(videoType, key, resourceList, page, pageSize, false);

            if (page == 1 && (searchResult == null || searchResult.videoInfoList == null || searchResult.videoInfoList.size() == 0)) {
                searchResult = searchAlbum(videoType, key, resourceList, page, pageSize, true);
            }

            debugLogger.info(String.format("搜索-solr专辑搜索耗时：（%s）-%s",key,(System.currentTimeMillis() - startt)));
            startt = System.currentTimeMillis();
            if (searchResult != null && searchResult.videoInfoList != null) {
                localList.addAll(searchResult.videoInfoList);
            }
@@ -457,11 +494,18 @@
                        }
                    }
                } catch (Exception e) {

                    debugLogger.error("常规搜索错误：{}","",e);
                }finally {
                    debugLogger.info(String.format("搜索-solr常规搜索耗时：（%s）-%s",key,(System.currentTimeMillis() - startt)));
                    startt = System.currentTimeMillis();
                }


            }
            System.out.println("搜索耗时:" + (System.currentTimeMillis() - startt));

            startt = System.currentTimeMillis();
            filterSearchResult(localList, resourceList);
            debugLogger.info(String.format("搜索-过滤耗时：（%s）-%s",key,(System.currentTimeMillis() - startt)));
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
@@ -469,10 +513,10 @@
        if (localList != null && localList.size() > 0) {
            list.addAll(localList);
        }

        startt = System.currentTimeMillis();
        //获取附加信息
        list = videoInfoExtraService.batchExtra(list, resourceList);

        debugLogger.info(String.format("搜索-附加信息耗时：（%s）-%s",key,(System.currentTimeMillis() - startt)));
        return new VideoListResultVO(list, videoType == 0 ? albumCount : 1000L);
    }