风行网页版爬虫

admin

2024-09-05 ab35ac8b769b2d9816dffb33a64f2c6f7bd5dd6e

 src/test/java/com/hxh/spring/test/video/HanmiTest.java

@@ -1,14 +1,27 @@
package com.hxh.spring.test.video;


import com.yeshi.buwan.dao.VideoInfoDao;
import com.yeshi.buwan.dao.juhe.hanmi.HanmiShowDao;
import com.yeshi.buwan.dao.juhe.hanmi.HanmiShowEpisodeDao;
import com.yeshi.buwan.dao.juhe.hanmi.VideoHanmiMapDao;
import com.yeshi.buwan.dao.juhe.youku.InternetSearchVideoDao;
import com.yeshi.buwan.domain.ResourceVideo;
import com.yeshi.buwan.domain.video.InternetSearchVideo;
import com.yeshi.buwan.service.imp.ResourceVideoService;
import com.yeshi.buwan.service.imp.VideoInfoService;
import com.yeshi.buwan.service.imp.VideoService;
import com.yeshi.buwan.service.inter.juhe.HanmiService;
import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService;
import com.yeshi.buwan.videos.hanmi.HanmiApiUtil;
import com.yeshi.buwan.videos.hanmi.entity.HanmiShow;
import com.yeshi.buwan.videos.hanmi.entity.HanmiShowEpisode;
import com.yeshi.buwan.videos.hanmi.entity.VideoHanmiMap;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
@@ -33,28 +46,69 @@
    @Resource
    private HanmiShowDao hanmiShowDao;

    final String MOVIE_URL = "https://www.wztaichuan.com/vod/show/area/%E9%9F%A9%E5%9B%BD/id/1/page/{页码}.html";
    final String SHOW_URL = "https://www.wztaichuan.com/vod/show/area/%E9%9F%A9%E5%9B%BD/id/3/page/{页码}.html";
    final String TV_URL = "https://www.wztaichuan.com/vod/type/id/5/page/{页码}.html";

    private  List<HanmiShow> updateShow(String listUrl,int page) throws Exception {
        List<HanmiShow> showList = HanmiApiUtil.parseDetailList(HanmiApiUtil.parseList(listUrl.replace("{页码}",page+"")));
        for (HanmiShow show : showList) {
            if(listUrl.contains("/id/1/")){
                show.setType("电影");
            }else if(listUrl.contains("/id/3/")){
                show.setType("综艺");
            }else if(listUrl.contains("/id/5/")){
                show.setType("电视剧");
            }
            hanmiService.saveShow(show);
        }
        return showList;
    }



    @Test
    public void updateLatest() throws Exception {
        String[] urls =new String[]{MOVIE_URL,SHOW_URL,TV_URL};
        for(String url: urls) {
            for (int i = 1; i < 20; i++) {
                System.out.printf(i + "==========================");
                // 电影 https://www.wztaichuan.com/vod/show/area/%E9%9F%A9%E5%9B%BD/id/1/page/2.html
                // 综艺 https://www.wztaichuan.com/vod/show/area/%E9%9F%A9%E5%9B%BD/id/3/page/2.html
                // 韩剧 https://www.wztaichuan.com/vod/type/id/5/page/2.html
                List<HanmiShow> showList =   updateShow(url,i);
                if (showList.size() < 10) {
                    break;
                }
            }
        }
    }


    @Test
    public void addHanJu() throws Exception {
        for (int i = 1; i < 3; i++) {
            List<HanmiShow> showList = HanmiApiUtil.parseDetailList(HanmiApiUtil.parseList("https://www.hmtv.me/hanju/page/" + i));
            for (HanmiShow show : showList) {
                hanmiService.saveShow(show);
//                hanmiService.deleteByShowId(show.getId());
        for (int i = 1; i < 30; i++) {
            List<HanmiShow> showList =   updateShow(SHOW_URL,i);
            if (showList.size() < 10) {
                break;
            }
        }
    }

    @Test
    public void addToVideo() throws IOException {

        for (int i = 1; i < 3; i++) {
            List<HanmiShow> showList = HanmiApiUtil.parseList("https://www.hmtv.me/hanju/page/" + i);
            Collections.reverse(showList);
            for (HanmiShow show : showList) {
                show.setId(show.getUrl().replace("/show/", "").trim());
        long count = hanmiService.countAll();
        int pageSize = 100;
        int totalPage=(int) (count%pageSize ==0?count/pageSize:count/pageSize+1);
        for(int i=0;i<totalPage;i++) {
            List<HanmiShow> list = hanmiService.listAll(i + 1, pageSize);
            for(HanmiShow show:list){
                try {
                show = hanmiService.getShowDetail(show.getId());
                if (show != null)
                    hanmiService.addToVideoInfo(show);
                    hanmiService.addToInternetSearch(show);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
@@ -74,29 +128,77 @@
            show = hanmiService.getShowDetail(show.getId());
            show.setType("韩剧");
            if (show != null)
                hanmiService.addToVideoInfo(show);
                hanmiService.addToInternetSearch(show);
        }

    }

    @Resource
    private VideoHanmiMapDao videoHanmiMapDao;

    @Resource
    private InternetSearchVideoDao internetSearchVideoDao;

    @Resource
    private ResourceVideoService resourceVideoService;

    @Resource
    private VideoInfoDao videoInfoDao;

    @Test
    public void update() {
        List<HanmiShow> list = hanmiShowDao.listAll(0, 100);
        for (int i = 0; i < 100; i++) {
            List<HanmiShow> list = hanmiShowDao.listAll(i*100, 100);
        for (HanmiShow show : list) {
            List<HanmiShowEpisode> epList = hanmiShowEpisodeDao.listByShowId(show.getId(), 0, 1);
            if (epList != null && epList.size() > 0) {
                try {
                    List<HanmiShowEpisode> episodeList = HanmiApiUtil.getShowEpisodesFromPlayUrl(epList.get(0).getPlayUrl());
                    if (episodeList != null)
                        for (HanmiShowEpisode episode : episodeList) {
                    show = HanmiApiUtil.parseShowDetail(show);
                    if (show != null && show.getEpisodeList() != null) {
                        // 更新封面图片
                        VideoHanmiMap map =  videoHanmiMapDao.selectByShowId(show.getId());
                        if(map!=null){
                            InternetSearchVideo video =     internetSearchVideoDao.get(map.getVideoId());
                            if(video!=null){
                                InternetSearchVideo update = new InternetSearchVideo();
                                update.setId(video.getId());
                                update.setVpicture(show.getPicture());
                                internetSearchVideoDao.updateSelective(update);
                            }
                        }
                        Query query = new Query();
                        query.addCriteria(new Criteria().where("_id").is(show.getId()));
                        Update update = new Update();
                        update.set("tag", show.getTag());
                        hanmiShowDao.update(query, update);
                        query = new Query();
                        query.addCriteria(new Criteria().where("showId").is(show.getId()));
                        hanmiShowEpisodeDao.delete(query);
                        // 删除原来的数据
                        for (HanmiShowEpisode episode : show.getEpisodeList()) {
                            episode.setShowId(show.getId());
                            episode.setId(HanmiShowEpisode.createId(episode.getShowId(), episode.getTag()));
                            episode.setCreateTime(new Date());
                            hanmiShowEpisodeDao.save(episode);
                        }
                } catch (IOException e) {
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }

            }
        }
    }


    @Test
    public void deleteFromVideo(){
        // 清除韩迷数据
        List<ResourceVideo>  resourceVideos =   resourceVideoService.listResourceVideo(28L,1,100 );
        for(ResourceVideo rv:resourceVideos){
            List<ResourceVideo> resourceList =   resourceVideoService.getResourceList(rv.getVideo().getId());
            resourceVideoService.delete(rv.getVideo().getId(), rv.getResource().getId());
            if(resourceList.size()<2){
                // 删除视频
                videoInfoDao.delete(rv.getVideo());
            }
        }
    }