爱奇艺资源更新优化

admin

2020-10-24 8075103992e9482fddd4a579d97b9a50c5dcf550

爱奇艺资源更新优化

 src/main/java/com/yeshi/buwan/dao/juhe/iqiyi/IqiyiAlbum2Dao.java

@@ -10,6 +10,7 @@
import com.yeshi.buwan.dao.base.MongodbBaseDao;
import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2;
import com.yeshi.buwan.iqiyi.entity.IqiyiVideoInfo;
import com.yeshi.buwan.query.Iqiyi2AlbumQuery;
import com.yeshi.buwan.util.StringUtil;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.query.Criteria;
@@ -17,24 +18,70 @@
import org.springframework.stereotype.Repository;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

@Repository
public class IqiyiAlbum2Dao extends MongodbBaseDao<IqiyiAlbum2> {

    //根据专辑ID查询
    public List<IqiyiAlbum2> listByAid(long aid, int sortName, int start, int count) {
    public List<IqiyiAlbum2> listByAid(Iqiyi2AlbumQuery albumQuery, int sortName, int start, int count) {
        Query query = new Query();
        query.addCriteria(Criteria.where("featureAlbumId").is(aid).andOperator(Criteria.where("contentType").is(1)));
        Criteria where = new Criteria();
        List<Criteria> whereList = new ArrayList<>();
        if (albumQuery.getFeatureAlbumId() != null)
            whereList.add(Criteria.where("featureAlbumId").is(albumQuery.getFeatureAlbumId()));
        if (albumQuery.getContentType() != null)
            whereList.add(Criteria.where("contentType").is(albumQuery.getContentType()));

        if (albumQuery.getChannelId() != null)
            whereList.add(Criteria.where("channelId").is(albumQuery.getChannelId()));
        if (whereList.size() > 0) {
            Criteria[] arrays = new Criteria[whereList.size()];
            whereList.toArray(arrays);
            where = where.andOperator(arrays);
        }

        query.addCriteria(where);
        query.skip(start);
        query.limit(count);
        List<Sort.Order> orders = new ArrayList<>();
        if (IqiyiAlbum2.SORT_ORDER == sortName)
            orders.add(new Sort.Order(Sort.Direction.ASC, "order"));
        else
        else if (IqiyiAlbum2.SORT_PERIOD == sortName)
            orders.add(new Sort.Order(Sort.Direction.DESC, "period"));
        else if (IqiyiAlbum2.SORT_ID == sortName) {
            orders.add(new Sort.Order(Sort.Direction.ASC, "id"));
        }

        query.with(new Sort(orders));
        return findList(query);
    }

    /**
     * æ ¹æ®Aid查询数量
     *
     * @param albumQuery
     * @return
     */
    public long countByAid(Iqiyi2AlbumQuery albumQuery) {
        Query query = new Query();
        Criteria where = new Criteria();
        List<Criteria> whereList = new ArrayList<>();
        if (albumQuery.getFeatureAlbumId() != null)
            whereList.add(Criteria.where("featureAlbumId").is(albumQuery.getFeatureAlbumId()));
        if (albumQuery.getContentType() != null)
            whereList.add(Criteria.where("contentType").is(albumQuery.getContentType()));

        if (albumQuery.getChannelId() != null)
            whereList.add(Criteria.where("channelId").is(albumQuery.getChannelId()));
        if (whereList.size() > 0) {
            Criteria[] arrays = new Criteria[whereList.size()];
            whereList.toArray(arrays);
            where = where.andOperator(arrays);
        }
        query.addCriteria(where);
        return count(query);
    }

    /**
@@ -112,4 +159,22 @@
    }


    /**
     * æ ¹æ®ID查询
     *
     * @param ids
     * @return
     */
    public List<IqiyiAlbum2> listByIds(List<Long> ids) {
        Query query = new Query();
        List<Criteria> whereList = new ArrayList<>();
        for (Long id : ids)
            whereList.add(Criteria.where("id").is(id));
        Criteria[] wheres = new Criteria[whereList.size()];
        whereList.toArray(wheres);
        query.addCriteria(new Criteria().orOperator(wheres));
        return findList(query);
    }


}

 src/main/java/com/yeshi/buwan/dto/mq/CMQConsumeMsg.java

New file
@@ -0,0 +1,23 @@
package com.yeshi.buwan.dto.mq;

//消息消费
public class CMQConsumeMsg {
    private String handler;
    private Object data;

    public String getHandler() {
        return handler;
    }

    public void setHandler(String handler) {
        this.handler = handler;
    }

    public Object getData() {
        return data;
    }

    public void setData(Object data) {
        this.data = data;
    }
}

 src/main/java/com/yeshi/buwan/iqiyi/IqiYiNewAPI.java

@@ -17,10 +17,10 @@

public class IqiYiNewAPI {

    public static int TYPE_DIANYING = 1;
    public static int TYPE_DIANSHIJU = 2;//电视剧
    public static int TYPE_ZONGYI = 6;//综艺
    public static int TYPE_DONGMAN = 4;//动漫
    public final static int TYPE_DIANYING = 1;
    public final static int TYPE_DIANSHIJU = 2;//电视剧
    public final static int TYPE_ZONGYI = 6;//综艺
    public final static int TYPE_DONGMAN = 4;//动漫

    public static int TYPE_YULE = 7;//娱乐
    public static int TYPE_KEJI = 30;//科技

 src/main/java/com/yeshi/buwan/iqiyi/entity/IqiyiAlbum2.java

@@ -14,6 +14,8 @@
    //按period字段排序
    public final static int SORT_PERIOD=2;

    public final static int SORT_ID=3;

    /**
     * id : 9000000001266601
     * featureAlbumId : 0

 src/main/java/com/yeshi/buwan/iqiyi/util/IqiyiUtil2.java

@@ -19,9 +19,7 @@

import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.*;

@Component
public class IqiyiUtil2 {
@@ -40,9 +38,10 @@
        for (IqiyiAlbum2 album : list) {
            if (album.getContentType() != 1)
                continue;
            if (!albumIsValid(album))
            if (!albumIsValid(album)) {//如果是无效的专辑就需要删除
                iqiyi2Service.deleteByAid(album.getId());
                continue;

            }
            if (album.getTvQipuIds() != null && album.getTvQipuIds().size() > 0) {
                //查询最新的一集是否缓存
                boolean needSaveVideos = true;
@@ -50,6 +49,7 @@
                    if (iqiyi2Service.countById(album.getLatestVideo().getId()) > 0L)
                        needSaveVideos = false;
                }


                if (needSaveVideos) {
                    int pageSize = 10;
@@ -59,13 +59,22 @@
                        int toIndex = (startIndex + pageSize) > album.getTvQipuIds().size() ? album.getTvQipuIds().size() : (startIndex + pageSize);
                        List<Long> tvQipuIds = new ArrayList<>();
                        tvQipuIds.addAll(album.getTvQipuIds().subList(startIndex, toIndex));

                        //查询是否已经存在了
                        List<IqiyiAlbum2> existList = iqiyi2Service.listByIds(tvQipuIds);
                        Set<Long> sets = new HashSet<>();
                        if (existList != null && existList.size() > 0) {
                            for (IqiyiAlbum2 a : existList)
                                sets.add(a.getId());

                            //删除已经存在的
                        for (int i = 0; i < tvQipuIds.size(); i++) {
                            long count = iqiyi2Service.countById(tvQipuIds.get(i));
                            if (count > 0) {
                                tvQipuIds.remove(i);
                                i--;
                                if (sets.contains(tvQipuIds.get(i))) {
                                    tvQipuIds.remove(i--);
                            }
                        }
                        }

                        if (tvQipuIds.size() > 0) {
                            List<IqiyiAlbum2> detailList = IqiYiNewAPI.getAlbumOrVideoDetail(tvQipuIds);
                            if (detailList != null)
@@ -132,12 +141,12 @@
    }

    //更新专辑
    public void updateAlbum(int categoryId, Date startTime, Date endTime) {
        IqiyiAlbumListResult result = IqiYiNewAPI.getUpdateAlbumList(categoryId + "", startTime, endTime, null, true, 10);
    public void updateAlbum(int categoryId, boolean isAlbum, Date startTime, Date endTime) {
        IqiyiAlbumListResult result = IqiYiNewAPI.getUpdateAlbumList(categoryId + "", startTime, endTime, null, isAlbum, 10);
        if (result.getAlbum2List() != null)
            saveAlbumAndVideo(result.getAlbum2List());
        while (result.getMinId() != null) {
            result = IqiYiNewAPI.getUpdateAlbumList(categoryId + "", startTime, endTime, result.getMinId(), true, 10);
            result = IqiYiNewAPI.getUpdateAlbumList(categoryId + "", startTime, endTime, result.getMinId(), isAlbum, 10);
            if (result.getAlbum2List() != null)
                saveAlbumAndVideo(result.getAlbum2List());
        }
@@ -157,25 +166,22 @@


    //更新最近一天的的专辑与视频
    public void updateTodayAlbumAndVideo() {
    public void updateTodayAlbumAndVideo(int channelId) {
        long now = System.currentTimeMillis();
        for (int i = 0; i < 8; i++) {
            Date endTime = new Date(now - i * 1000 * 60 * 60L * 3);
            Date endTime = new Date(now - 1000 * 60 * 60L * 3 * i);
            Date startTime = new Date(endTime.getTime() - 1000 * 60 * 60L * 3);

            //更新电影
            updateVideo(IqiYiNewAPI.TYPE_DIANYING, startTime, endTime);
            //更新电视剧
            updateAlbum(IqiYiNewAPI.TYPE_DIANSHIJU, startTime, endTime);
            updateVideo(IqiYiNewAPI.TYPE_DIANSHIJU, startTime, endTime);

            //更新动漫
            updateAlbum(IqiYiNewAPI.TYPE_DONGMAN, startTime, endTime);
            updateVideo(IqiYiNewAPI.TYPE_DONGMAN, startTime, endTime);

            //更新综艺
            updateAlbum(IqiYiNewAPI.TYPE_ZONGYI, startTime, endTime);
            updateVideo(IqiYiNewAPI.TYPE_ZONGYI, startTime, endTime);
            switch (channelId) {
                case IqiYiNewAPI.TYPE_DIANYING:
                    updateAlbum(channelId, false, startTime, endTime);
                    break;
                case IqiYiNewAPI.TYPE_DIANSHIJU:
                case IqiYiNewAPI.TYPE_DONGMAN:
                case IqiYiNewAPI.TYPE_ZONGYI:
                    updateAlbum(channelId, true, startTime, endTime);
                    break;
            }
        }
    }


 src/main/java/com/yeshi/buwan/job/SMSJob.java

New file
@@ -0,0 +1,68 @@
package com.yeshi.buwan.job;

import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.handler.annotation.XxlJob;
import com.yeshi.buwan.log.LogHelper;
import com.yeshi.buwan.util.HttpUtil;
import com.yeshi.buwan.util.StringUtil;
import net.sf.json.JSONObject;
import org.springframework.stereotype.Component;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;

@Component
public class SMSJob {

    /**
     * å‘送聚合短信
     *
     * @param mobiles
     * @throws Exception
     */
    public void sendJuHeSMS(List<String> mobiles) throws Exception {
        String url = String.format("http://v.juhe.cn/shortLetter/send.php?key=%s&tplId=%s&mobile=%s", "3f1ed706b36412dec6b5cd3cf1d0e5e3", "4558", URLEncoder.encode(StringUtil.join(mobiles, ",")));
        String result = HttpUtil.get(url);
        JSONObject resultJSON = JSONObject.fromObject(result);
        if (resultJSON.optInt("error_code") != 0) {
            throw new Exception(result);
        }
    }

    /**
     * èŽ·å–电话号码
     *
     * @return
     */
    private List<String> getMobiles() {
        Scanner scanner = new Scanner(SMSJob.class.getClassLoader().getResourceAsStream("电话.txt"));
        List<String> mobileList = new ArrayList<>();
        while (scanner.hasNextLine()) {
            String phone = scanner.nextLine();
            if (!StringUtil.isNullOrEmpty(phone)) {
                mobileList.add(phone.trim());
            }
        }
        scanner.close();
        return mobileList;
    }

    @XxlJob("sms-s11hongbao")
    public ReturnT<String> sendSMS(String param) throws Exception {
        List<String> phoneList = getMobiles();
        int page = phoneList.size() % 1000 == 0 ? phoneList.size() / 1000 : phoneList.size() / 1000 + 1;
        for (int i = 0; i < page; i++) {
            List<String> phones = phoneList.subList(1000 * i, (1000 * i + 1000) >= phoneList.size() ? phoneList.size() : (1000 * i + 1000));
            try {
                sendJuHeSMS(phones);
            } catch (Exception e) {
                LogHelper.error("短信发送出错，page：" + i);
            }
        }
        return ReturnT.SUCCESS;
    }


}

 src/main/java/com/yeshi/buwan/job/video/Iqiyi2VideoUpdate.java

@@ -9,6 +9,7 @@
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.util.Date;
import java.util.List;

@Component

 src/main/java/com/yeshi/buwan/query/Iqiyi2AlbumQuery.java

New file
@@ -0,0 +1,31 @@
package com.yeshi.buwan.query;

public class Iqiyi2AlbumQuery {
    private Long featureAlbumId;
    private Integer channelId;
    private Integer contentType;

    public Long getFeatureAlbumId() {
        return featureAlbumId;
    }

    public void setFeatureAlbumId(Long featureAlbumId) {
        this.featureAlbumId = featureAlbumId;
    }

    public Integer getChannelId() {
        return channelId;
    }

    public void setChannelId(Integer channelId) {
        this.channelId = channelId;
    }

    public Integer getContentType() {
        return contentType;
    }

    public void setContentType(Integer contentType) {
        this.contentType = contentType;
    }
}

 src/main/java/com/yeshi/buwan/service/imp/ResourceVideoService.java

@@ -4,6 +4,7 @@
import java.util.List;

import com.yeshi.buwan.domain.VideoResource;
import com.yeshi.buwan.util.mq.CMQManager;
import org.hibernate.HibernateException;
import org.hibernate.Query;
import org.hibernate.Session;
@@ -14,6 +15,7 @@
import com.yeshi.buwan.dao.ResourceVideoDao;
import com.yeshi.buwan.domain.ResourceVideo;
import com.yeshi.buwan.domain.VideoInfo;
import org.springframework.transaction.annotation.Transactional;

@Service
public class ResourceVideoService {
@@ -53,6 +55,7 @@

    /**
     * èŽ·å–资源列表
     *
     * @param videoId
     * @return
     */
@@ -81,5 +84,13 @@
        resourceVideoDao.delete(rv);
    }

    @Transactional
    public void delete(String videoId, String resourceId) {
        ResourceVideo resourceVideo = resourceVideoDao.selectByVideoIdAndResourceId(videoId, resourceId + "");
        if (resourceVideo != null)
            resourceVideoDao.delete(resourceVideo);
        CMQManager.getInstance().addVideoResourceDeleteMsg(videoId);
    }


}

 src/main/java/com/yeshi/buwan/service/imp/juhe/Iqiyi2ServiceImpl.java

@@ -12,6 +12,7 @@
import com.yeshi.buwan.iqiyi.entity.VideoIqiyi2;
import com.yeshi.buwan.iqiyi.util.IqiyiUtil;
import com.yeshi.buwan.iqiyi.util.IqiyiUtil2;
import com.yeshi.buwan.query.Iqiyi2AlbumQuery;
import com.yeshi.buwan.service.imp.CategoryVideoService;
import com.yeshi.buwan.service.imp.ResourceVideoService;
import com.yeshi.buwan.service.imp.VideoInfoService;
@@ -70,7 +71,10 @@
        } else {//专辑
            //判断period字段是否为空
            int sort = album.isSourceAlbum() ? IqiyiAlbum2.SORT_PERIOD : IqiyiAlbum2.SORT_ORDER;
            List<IqiyiAlbum2> list = iqiyiAlbum2Dao.listByAid(album.getId(), sort, (page - 1) * pageSize, pageSize);
            Iqiyi2AlbumQuery query = new Iqiyi2AlbumQuery();
            query.setFeatureAlbumId(album.getId());
            query.setContentType(1);
            List<IqiyiAlbum2> list = iqiyiAlbum2Dao.listByAid(query, sort, (page - 1) * pageSize, pageSize);
            if (list != null)
                album2List.addAll(list);
        }
@@ -334,4 +338,24 @@
    public long countById(Long id) {
        return iqiyiAlbum2Dao.countById(id);
    }

    @Override
    public void deleteByAid(long id) {
        long count = iqiyiAlbum2Dao.countById(id);
        if (count > 0) {
            iqiyiAlbum2Dao.delete(id);
        }
        VideoIqiyi2 bean = videoIqiyi2Dao.selectByIqiyiId(id);
        if (bean != null) {
            videoIqiyi2Dao.delete(bean.getVideoId());
            //删除资源
            resourceVideoService.delete(bean.getVideoId() + "", IqiyiUtil2.RESOURCE_ID + "");
        }

    }

    @Override
    public List<IqiyiAlbum2> listByIds(List<Long> idsList) {
        return iqiyiAlbum2Dao.listByIds(idsList);
    }
}

 src/main/java/com/yeshi/buwan/service/inter/juhe/Iqiyi2Service.java

@@ -53,4 +53,18 @@
    public IqiyiAlbum2 selectAlbumById(Long id);

    public long countById(Long id);

    /**
     * æ ¹æ®ä¸“辑ID删除
     * @param id
     */
    public void deleteByAid(long id);


    /**
     * æ ¹æ®ID查询
     * @param idsList
     * @return
     */
    public List<IqiyiAlbum2> listByIds(List<Long> idsList);
}

 src/main/java/com/yeshi/buwan/util/SpringContext.java

@@ -1,6 +1,8 @@
package com.yeshi.buwan.util;

import com.yeshi.buwan.domain.ResourceVideo;
import com.yeshi.buwan.domain.VideoInfo;
import com.yeshi.buwan.dto.mq.CMQConsumeMsg;
import com.yeshi.buwan.dto.mq.FunTVAlbum2MQMsg;
import com.yeshi.buwan.dto.mq.IqiyiAlbum2MQMsg;
import com.yeshi.buwan.dto.mq.SolrVideoMQMsg;
@@ -8,6 +10,7 @@
import com.yeshi.buwan.funtv.entity.FunTVVideo2;
import com.yeshi.buwan.iqiyi.entity.IqiyiAlbum2;
import com.yeshi.buwan.service.imp.JobThreadExecutorServiceImpl;
import com.yeshi.buwan.service.imp.ResourceVideoService;
import com.yeshi.buwan.service.imp.VideoInfoService;
import com.yeshi.buwan.service.inter.juhe.FunTV2Service;
import com.yeshi.buwan.service.inter.juhe.Iqiyi2Service;
@@ -43,6 +46,9 @@
    @Resource
    private FunTV2Service funTV2Service;

    @Resource
    private ResourceVideoService resourceVideoService;


    private static boolean isInited = false;

@@ -68,6 +74,7 @@
            doSolrJob();
            doAddIqiyi2Video();
            doAddFunTV2Video();
            doDeleteVideoResource();
        }
    }

@@ -145,5 +152,34 @@

    }

    private void doDeleteVideoResource() {
        new JobThreadExecutorServiceImpl().run(new Runnable() {
            @Override
            public void run() {
                while (true) {
                    List<CMQConsumeMsg> cmqMsgList = CMQManager.getInstance().consumeVideoResourceDeleteMsg(16);
                    if (cmqMsgList != null)
                        for (CMQConsumeMsg msg : cmqMsgList) {
                            try {
                                String videoId = msg.getData() + "";
                                //查询资源列表
                                List<ResourceVideo> resourceVideoList = resourceVideoService.getResourceList(videoId);
                                //隐藏视频
                                if (resourceVideoList == null || resourceVideoList.size() == 0)
                                    videoInfoService.hiddenVideo(videoId);
                                //更新搜索引擎
                                CMQManager.getInstance().addSolrMsg(videoId);
                                CMQManager.getInstance().deleteVideoResourceDeleteMsg(msg.getHandler());
                            } catch (Exception e) {
                                logger.error("视频资源删除处理出错：" + e.getMessage());
                                logger.error("ID:" + msg.getData());
                            }
                        }
                }
            }
        });

    }


}

 src/main/java/com/yeshi/buwan/util/mq/CMQManager.java

@@ -1,6 +1,7 @@
package com.yeshi.buwan.util.mq;

import com.qcloud.cmq.Message;
import com.yeshi.buwan.dto.mq.CMQConsumeMsg;
import com.yeshi.buwan.dto.mq.FunTVAlbum2MQMsg;
import com.yeshi.buwan.dto.mq.IqiyiAlbum2MQMsg;
import com.yeshi.buwan.dto.mq.SolrVideoMQMsg;
@@ -20,6 +21,9 @@
    //视频更新-风行2
    public static String QUEUENAME_VIDEO_UPDATE_FUNTV_2 = "buwan-video-update-funtv2";

    //删除视频资源
    public static String QUEUENAME_VIDEO_RESOURCE_DELETE = "buwan-video-resource-delete";


    static {
        cmqUtil = CMQUtil.getInstance(secretId, secretKey);
@@ -27,6 +31,7 @@
        cmqUtil.createQueue(QUEUENAME_SOLR, 1024 * 1024);
        cmqUtil.createQueue(QUEUENAME_VIDEO_UPDATE_IQIYI_2, 1024 * 1024);
        cmqUtil.createQueue(QUEUENAME_VIDEO_UPDATE_FUNTV_2, 1024 * 1024);
        cmqUtil.createQueue(QUEUENAME_VIDEO_RESOURCE_DELETE, 1024 * 1024);
    }

    public static CMQManager getInstance() {
@@ -118,4 +123,34 @@
        cmqUtil.deleteMsg(QUEUENAME_SOLR, handler);
    }


    /**
     * åˆ é™¤è§†é¢‘资源
     *
     * @param videoId
     */

    public void addVideoResourceDeleteMsg(String videoId) {
        cmqUtil.sendMsg(QUEUENAME_VIDEO_RESOURCE_DELETE, videoId);
    }

    public List<CMQConsumeMsg> consumeVideoResourceDeleteMsg(int count) {
        List<CMQConsumeMsg> list = new ArrayList<>();
        List<Message> msgList = cmqUtil.recieveMsg(count, QUEUENAME_VIDEO_RESOURCE_DELETE);
        if (msgList != null)
            for (Message msg : msgList) {
                CMQConsumeMsg mm = new CMQConsumeMsg();
                mm.setHandler(msg.receiptHandle);
                mm.setData(msg.msgBody);
                list.add(mm);
            }
        return list;
    }


    public void deleteVideoResourceDeleteMsg(String handler) {
        cmqUtil.deleteMsg(QUEUENAME_VIDEO_RESOURCE_DELETE, handler);
    }


}

 src/main/resources/µç»°.txt

New file
Diff too large