爱奇艺资源优化

admin

2020-10-23 8b2c4df38c2ea983be4db3aa4e18830339537806

爱奇艺资源优化

 src/main/java/com/yeshi/buwan/dao/juhe/iqiyi/IqiyiAlbum2Dao.java

@@ -1,5 +1,12 @@
package com.yeshi.buwan.dao.juhe.iqiyi;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.yeshi.buwan.dao.base.MongodbBaseDao;
import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2;
import com.yeshi.buwan.iqiyi.entity.IqiyiVideoInfo;
@@ -30,6 +37,18 @@
        return findList(query);
    }

    /**
     * 根据专辑名称计数
     *
     * @param aid
     * @return
     */
    public long countVideoByAid(long aid) {
        Query query = new Query();
        query.addCriteria(Criteria.where("featureAlbumId").is(aid).andOperator(Criteria.where("contentType").is(1)));
        return count(query);
    }

    public List<IqiyiAlbum2> listByChannelId(int channelId, Long albumId, Integer contentType, int start, int count) {
        Query query = new Query();
        List<Criteria> andCriteria = new ArrayList<>();
@@ -49,5 +68,48 @@
        return findList(query);
    }

    /**
     * 根据ID查询
     *
     * @param id
     * @return
     */
    public long countById(Long id) {
        Query query = new Query();
        query.addCriteria(Criteria.where("id").is(id));
        return count(query);
    }

    /**
     * 获取无效列表
     *
     * @param start
     * @param count
     * @return
     */
    public List<Long> listInvalid(int start, int count) {
        List<Long> idsList = new ArrayList<>();

        BasicDBObject where = new BasicDBObject();
        where.put("platformId", 15);
        where.put("availableStatus", new BasicDBObject("$ne", 1));

        BasicDBObject query = new BasicDBObject("playControls", new BasicDBObject("$elemMatch", where));
        BasicDBList condList = new BasicDBList();
        condList.add(query);
        condList.add(new BasicDBObject("effect", new BasicDBObject("$ne", 1)));
        condList.add(new BasicDBObject("supportDrm", new BasicDBObject("$eq", true)));
        query = new BasicDBObject("$or", condList);
        DBCursor cursor = mongoTemplate.getCollection(mongoTemplate.getCollectionName(IqiyiAlbum2.class)).find(query, new BasicDBObject("_id", 1)).skip(start).limit(count);
        if (cursor != null && cursor.hasNext() && cursor.size() > 0) {
            String json = JSON.toJSONString(cursor, true);
            JSONArray array = JSON.parseArray(json);
            for (int i = 0; i < array.size(); i++) {
                idsList.add(array.getJSONObject(i).getLong("_id"));
            }
        }
        return idsList;
    }


}

 src/main/java/com/yeshi/buwan/iqiyi/util/IqiyiUtil2.java

@@ -38,16 +38,41 @@
        if (list == null)
            return;
        for (IqiyiAlbum2 album : list) {
            if (album.getContentType() != 1)
                continue;
            if (!albumIsValid(album))
                continue;

            if (album.getTvQipuIds() != null && album.getTvQipuIds().size() > 0) {
                int pageSize = 10;
                int page = album.getTvQipuIds().size() % pageSize == 0 ? album.getTvQipuIds().size() / pageSize : album.getTvQipuIds().size() / pageSize + 1;
                for (int p = 0; p < page; p++) {
                    int startIndex = p * pageSize;
                    int toIndex = (startIndex + pageSize) > album.getTvQipuIds().size() ? album.getTvQipuIds().size() : (startIndex + pageSize);
                    List<IqiyiAlbum2> detailList = IqiYiNewAPI.getAlbumOrVideoDetail(album.getTvQipuIds().subList(startIndex, toIndex));
                    if (detailList != null)
                        for (IqiyiAlbum2 video : detailList)
                            iqiyi2Service.saveIqiyiAlbum(video);
                //查询最新的一集是否缓存
                boolean needSaveVideos = true;
                if (album.getLatestVideo() != null) {
                    if (iqiyi2Service.countById(album.getLatestVideo().getId()) > 0L)
                        needSaveVideos = false;
                }

                if (needSaveVideos) {
                    int pageSize = 10;
                    int page = album.getTvQipuIds().size() % pageSize == 0 ? album.getTvQipuIds().size() / pageSize : album.getTvQipuIds().size() / pageSize + 1;
                    for (int p = 0; p < page; p++) {
                        int startIndex = p * pageSize;
                        int toIndex = (startIndex + pageSize) > album.getTvQipuIds().size() ? album.getTvQipuIds().size() : (startIndex + pageSize);
                        List<Long> tvQipuIds = new ArrayList<>();
                        tvQipuIds.addAll(album.getTvQipuIds().subList(startIndex, toIndex));
                        for (int i = 0; i < tvQipuIds.size(); i++) {
                            long count = iqiyi2Service.countById(tvQipuIds.get(i));
                            if (count > 0) {
                                tvQipuIds.remove(i);
                                i--;
                            }
                        }
                        if (tvQipuIds.size() > 0) {
                            List<IqiyiAlbum2> detailList = IqiYiNewAPI.getAlbumOrVideoDetail(tvQipuIds);
                            if (detailList != null)
                                for (IqiyiAlbum2 video : detailList)
                                    iqiyi2Service.saveIqiyiAlbum(video);
                        }
                    }
                }
            }
            iqiyi2Service.saveIqiyiAlbum(album);
@@ -88,12 +113,12 @@

    //拉取所有的电影
    public void syncAllDianYing() {
        syncAlbumAndVideo(IqiYiNewAPI.TYPE_DIANYING, false, 255675400l);
        syncAlbumAndVideo(IqiYiNewAPI.TYPE_DIANYING, false, null);
    }

    //拉取所有的电视剧
    public void syncAllDianShiJu() {
        syncAlbumAndVideo(IqiYiNewAPI.TYPE_DIANSHIJU, true, 206929801L);
        syncAlbumAndVideo(IqiYiNewAPI.TYPE_DIANSHIJU, true, null);
    }

    //拉取所有的动漫
@@ -211,7 +236,7 @@
            else
                vi.setTag(album.getOrder() + "");
        } else if (album.getChannelId() == IqiYiNewAPI.TYPE_ZONGYI)
            vi.setTag(album.getPeriod());
            vi.setTag(album.getPeriod() + "\n" + album.getSubTitle());
        vi.setType("album");
        return vi;
    }
@@ -257,5 +282,28 @@
        return list;
    }

    /**
     * 专辑是否有效
     *
     * @param album2
     * @return
     */
    public static boolean albumIsValid(IqiyiAlbum2 album2) {
        if (album2.getEffect() != 1)
            return false;

        if (album2.isSupportDrm())
            return false;

        List<IqiyiAlbum2.PlayControlsBean> playControls = album2.getPlayControls();
        if (playControls == null)
            return false;
        for (IqiyiAlbum2.PlayControlsBean bean : playControls) {
            if (bean.getPlatformId() == 15 && bean.getAvailableStatus() == 1)
                return true;
        }
        return false;
    }


}

 src/main/java/com/yeshi/buwan/service/imp/CategoryVideoService.java

@@ -21,7 +21,7 @@

    public void addCategoryVideo(String videoId, long categoryId) {
        List<CategoryVideo> list = categoryVideoDao.listByVideoIdAndCategoryId(videoId, categoryId);
        if (list == null && list.size() == 0) {
        if (list == null || list.size() == 0) {
            CategoryVideo cv = new CategoryVideo();
            cv.setVideo(new VideoInfo(videoId));
            cv.setVideoType(new VideoType(categoryId));

 src/main/java/com/yeshi/buwan/service/imp/juhe/Iqiyi2ServiceImpl.java

@@ -127,16 +127,17 @@
                return;
        }

        if (album.getChannelId() != 1) {

        if (album.getChannelId() != 1 && !VideoConstant.iqiyiSpecialNames.contains(album.getName())) {
            //标题中只能包含中英文与数字
            String regx = "^[(\\u4e00-\\u9fa5)( )a-zA-Z0-9]+$";
            String regx = "^[(\\u4e00-\\u9fa5)(：)( )a-zA-Z0-9]+$";
            if (!Pattern.matches(regx, album.getName())) {
                return;
            }
        }

        //过滤某些短片与影评
        if (album.getName().contains("《"))
        if (album.getName().contains("《") && !VideoConstant.iqiyiSpecialNames.contains(album.getName()))
            return;

        //空电视剧或者是空动漫则返回
@@ -153,7 +154,7 @@
            if (videoInfo == null)
                return;
            newVideoInfo.setId(videoInfo.getId());
            if (!newVideoInfo.getTag().equalsIgnoreCase(videoInfo.getTag())) {
            if (!newVideoInfo.getTag().equalsIgnoreCase(videoInfo.getTag()) || newVideoInfo.getVideocount() != videoInfo.getVideocount()) {
                newVideoInfo.setCreatetime(videoInfo.getCreatetime());
                //更新信息tag信息
                videoInfo.setLatestHpicture(newVideoInfo.getLatestHpicture());
@@ -165,6 +166,7 @@
                videoInfo.setVpicture(newVideoInfo.getVpicture());
                videoInfo.setShow(newVideoInfo.getShow());
                videoInfo.setVideoType(newVideoInfo.getVideoType());
                videoInfo.setUpdatetime(newVideoInfo.getUpdatetime());
                videoInfoDao.update(videoInfo);
            }
        } else {//视频不存在
@@ -186,6 +188,7 @@
                oldVideo.setVpicture(newVideoInfo.getVpicture());
                oldVideo.setShow(newVideoInfo.getShow());
                oldVideo.setVideoType(newVideoInfo.getVideoType());
                oldVideo.setUpdatetime(newVideoInfo.getUpdatetime());
                videoInfoDao.update(oldVideo);
            }

@@ -310,11 +313,11 @@
                video.setLatestVpicture(IqiyiUtil.getVPicture(album2.getImageUrl()));
            }
        }
        long videoCount = iqiyiAlbum2Dao.countVideoByAid(album.getId());
        if (videoCount == 0L)
            videoCount = 1;

        if (album.getTvQipuIds() != null && album.getTvQipuIds().size() > 0)
            video.setVideocount(album.getTvQipuIds().size());
        else
            video.setVideocount(1);
        video.setVideocount((int) videoCount);

        video.setUpdatetime(System.currentTimeMillis() + "");

@@ -326,4 +329,9 @@
    public IqiyiAlbum2 selectAlbumById(Long id) {
        return iqiyiAlbum2Dao.get(id);
    }

    @Override
    public long countById(Long id) {
        return iqiyiAlbum2Dao.countById(id);
    }
}

 src/main/java/com/yeshi/buwan/service/inter/juhe/Iqiyi2Service.java

@@ -15,7 +15,7 @@
     * @param videoId
     * @return
     */
    public List<VideoDetailInfo> getVideoDetailList(String videoId,int page,int pageSize);
    public List<VideoDetailInfo> getVideoDetailList(String videoId, int page, int pageSize);

    /**
     * 获取最新一集的详情
@@ -51,4 +51,6 @@


    public IqiyiAlbum2 selectAlbumById(Long id);

    public long countById(Long id);
}

 src/main/java/com/yeshi/buwan/util/StringUtil.java

@@ -40,9 +40,7 @@
        Matcher m = p.matcher(mobile);

        if (mobile == null || mobile.equals("") || mobile.length() != 11) {

            return false;

        } else {
            return m.find();
        }

 src/main/java/com/yeshi/buwan/util/video/VideoConstant.java

@@ -1,7 +1,17 @@
package com.yeshi.buwan.util.video;

import java.util.Arrays;
import java.util.List;

public class VideoConstant {


    public final static List<String> iqiyiSpecialNames = Arrays.asList(new String[]{
            //不参与过滤的词
            "《卧底》"
    });


    public final static int VIDEO_CATEGORY_DIANYING = 151;
    public final static int VIDEO_CATEGORY_DIANSHIJU = 150;
    public final static int VIDEO_CATEGORY_DONGMAN = 153;

 src/test/java/com/hxh/spring/test/Iqiyi2.java

@@ -1,10 +1,16 @@
package com.hxh.spring.test;

import com.google.gson.Gson;
import com.yeshi.buwan.dao.juhe.iqiyi.IqiyiAlbum2Dao;
import com.yeshi.buwan.iqiyi.IqiYiNewAPI;
import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2;
import com.yeshi.buwan.iqiyi.util.IqiyiUtil2;
import com.yeshi.buwan.iqiyi.vo.IqiyiAlbumListResult;
import com.yeshi.buwan.job.video.Iqiyi2VideoUpdate;
import com.yeshi.buwan.service.inter.juhe.Iqiyi2Service;
import com.yeshi.buwan.util.FileUtil;
import com.yeshi.buwan.util.StringUtil;
import com.yeshi.buwan.util.TimeUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
@@ -12,7 +18,12 @@
import org.springframework.test.context.web.WebAppConfiguration;

import javax.annotation.Resource;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

//
@@ -30,10 +41,13 @@
    @Resource
    private Iqiyi2Service iqiyi2Service;

    @Resource
    private Iqiyi2VideoUpdate iqiyi2VideoUpdate;


    @Test
    public void syncAll() {
        iqiyiUtil2.syncAllZongYi();
        iqiyiUtil2.syncAllDianShiJu();
    }

    @Test
@@ -43,7 +57,7 @@

    @Test
    public void addToVideoInfo() {
        List<IqiyiAlbum2> list = iqiyiAlbum2Dao.listByChannelId(1, null, 1,11000, 3000);
        List<IqiyiAlbum2> list = iqiyiAlbum2Dao.listByChannelId(1, null, 1, 11000, 3000);
        for (IqiyiAlbum2 album : list)
            try {
                iqiyi2Service.addToVideoInfo(album);
@@ -75,13 +89,11 @@
    @Test
    public void syncByAid() {

        Long[] ids = new Long[]{
                246508201L, 248886801L, 230798901L, 1569537751867101L,
//        Long[] ids = new Long[]{
//                246508201L, 248886801L, 230798901L, 1569537751867101L,
//        };


        };

        iqiyiUtil2.syncByAid(240793601L);
        iqiyiUtil2.syncByAid(7643537550826500L);

//        IqiyiAlbum2 album2 = iqiyi2Service.selectAlbumById(240793601L);
//        if (album2 != null) {
@@ -101,9 +113,54 @@

    @Test
    public void test6() {
        List<IqiyiAlbum2> list = iqiyiAlbum2Dao.listByChannelId(1, null, 1, 0, 1000);
        for (IqiyiAlbum2 album : list)
            System.out.println(album.getName());
        for(int i=0;i<10;i++) {
            List<Long> ids = iqiyiAlbum2Dao.listInvalid(1000*(i), 1000);
            saveToFile(ids);
        }
    }

    private void saveToFile(List<Long> list) {
        FileWriter fw = null;
        try {
            //如果文件存在，则追加内容；如果文件不存在，则创建文件
            File f = new File("E:\\iqiyialbum_invalid.txt");
            fw = new FileWriter(f, true);
        } catch (IOException e) {
            e.printStackTrace();
        }
        PrintWriter pw = new PrintWriter(fw);
        pw.println(new Gson().toJson(list));
        pw.flush();
        try {
            fw.flush();
            pw.close();
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }


    }

    @Test
    public void count() {
//        long count = iqiyiAlbum2Dao.countVideoByAid(253120001L);
//        System.out.println(count);
        IqiyiAlbum2 album2 = iqiyiAlbum2Dao.get(6603128446259700L);
        iqiyi2Service.addToVideoInfo(album2);
    }


    @Test
    public void sync() {

    }

    public static void main(String[] args) {
        Date startTime = new Date(TimeUtil.convertGernalTime("2020-10-23", "yyyy-MM-dd"));
        Date endTime = new Date(TimeUtil.convertGernalTime("2020-10-23 03", "yyyy-MM-dd HH"));
        IqiyiAlbumListResult result = IqiYiNewAPI.getUpdateAlbumList(IqiYiNewAPI.TYPE_DIANYING + "", startTime, endTime, null, false, 20);
        System.out.println(result);
    }