风行网页版爬虫

admin

2024-09-05 ab35ac8b769b2d9816dffb33a64f2c6f7bd5dd6e

 src/main/java/com/yeshi/buwan/job/video/TencentVideoUpdate.java

@@ -3,10 +3,11 @@
import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.handler.annotation.XxlJob;
import com.yeshi.buwan.service.inter.juhe.TencentVideoService;
import com.yeshi.buwan.util.StringUtil;
import com.yeshi.buwan.videos.tencent.TencentVideoApiUtil;
import com.yeshi.buwan.videos.tencent.entity.TencentCoverInfo;
import com.yeshi.buwan.util.StringUtil;
import com.yeshi.buwan.util.video.web.TencentWebUtil;
import com.yeshi.buwan.videos.tencent.factory.TencentCoverInfoFactory;
import com.yeshi.buwan.videos.tencent.vo.TencentCoverInfoVO;
import net.sf.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -18,52 +19,39 @@
@Component
public class TencentVideoUpdate {

    private final static Logger logger = LoggerFactory.getLogger(TencentVideoUpdate.class);
    private final static Logger logger = LoggerFactory.getLogger("debug");

    @Resource
    private TencentVideoService tencentVideoService;


    private TencentCoverInfo getCoverDetail(String coverId) throws Exception {
        TencentCoverInfo detail = TencentVideoApiUtil.getCoverInfo(String.format("https://v.qq.com/x/cover/%s.html", coverId));
        return detail;
        TencentCoverInfoVO detail = TencentVideoApiUtil.getCoverInfo(String.format("https://v.qq.com/x/cover/%s.html", coverId));
        return TencentCoverInfoFactory.create(detail);
    }

    private void updateCategory(String channel) {
        int totalPage = 50;
        for (int i = 0; i < totalPage; i++) {
            List<TencentCoverInfo> coverInfoList = TencentVideoApiUtil.getVideoListByCategory(channel, i + 1);
    public void updateCategory(String channel,int startPage,int endPage, Integer areaId) {
        logger.info("更新分类："+channel);
        for (int i = startPage; i <= endPage; i++) {
            List<TencentCoverInfoVO> coverInfoList = TencentVideoApiUtil.getVideoListByCategory(channel, i, areaId);
            save(coverInfoList);
        }
    }

    private void save(List<TencentCoverInfo> coverInfoList) {
    private void save(List<TencentCoverInfoVO> coverInfoList) {
        if (coverInfoList != null)
            for (TencentCoverInfo coverInfo : coverInfoList) {
            for (TencentCoverInfoVO coverInfo : coverInfoList) {
                System.out.println(coverInfo.getTitle() + ":" + coverInfo.getCover_id());
                try {
                    tencentVideoService.addToInternetSearch(coverInfo);
//                    tencentVideoService.save(TencentCoverInfoFactory.create(coverInfo));
                    tencentVideoService.addToInternetSearch( TencentCoverInfoFactory.create(coverInfo),true);
                } catch (Exception e) {
                    e.printStackTrace();
                    logger.error("保存出错",e);
                }
            }
    }

    private String getChannel(String cate) throws Exception {
        switch (cate) {
            case "综艺":
                return "variety";
            case "电影":
                return "movie";
            case "电视剧":
                return "tv";
            case "动漫":
                return "cartoon";
        }

        throw new Exception("类型不匹配");

    }

    /**
     * 更新最近几天的视频
@@ -79,10 +67,10 @@
        };

        if (!StringUtil.isNullOrEmpty(param)) {
            updateCategory(getChannel(param));
            updateCategory(param,1,1,null);
        } else {
            for (String type : types) {
                updateCategory(getChannel(type));
                updateCategory(type,1,1, null);
            }
        }
        return ReturnT.SUCCESS;
@@ -94,8 +82,8 @@
        JSONObject json = JSONObject.fromObject(param);
        int page = json.optInt("page");
        String url = json.optString("url");
        List<TencentCoverInfo> list = TencentVideoApiUtil.getVideoList(TencentWebUtil.getApiUrl(url, page));
        save(list);
//        List<TencentCoverInfoVO> list = TencentVideoApiUtil.getVideoList(TencentWebUtil.getApiUrl(url, page));
//        save(list);
        return ReturnT.SUCCESS;
    }

@@ -113,7 +101,7 @@
        for (String cid : cids) {
            try {
                TencentCoverInfo detail = getCoverDetail(cid);
                tencentVideoService.addToInternetSearch(detail);
                tencentVideoService.addToInternetSearch(detail, true);
            } catch (Exception e) {
                e.printStackTrace();
            }