src/main/java/com/yeshi/buwan/controller/MonitorController.java
@@ -2,7 +2,7 @@ import com.yeshi.buwan.dto.search.SolrResultDTO; import com.yeshi.buwan.dto.search.SolrVideoSearchFilter; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; src/main/java/com/yeshi/buwan/controller/admin/api/VideoAdminController.java
@@ -19,7 +19,7 @@ import com.yeshi.buwan.iqiyi.IqiYiNewAPI; import com.yeshi.buwan.service.imp.VideoResourceService; import com.yeshi.buwan.service.inter.video.VideoInfoExtraService; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.util.Constant; import com.yeshi.buwan.vo.video.VideoAdminInfoVO; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; src/main/java/com/yeshi/buwan/controller/api/ApiControllerV3.java
@@ -431,4 +431,17 @@ out.close(); } // 搜索类接口 @RequestMapping(value = "/search/{method}", method = RequestMethod.POST) public void search(AcceptData acceptData, @PathVariable String method, HttpServletRequest request, PrintWriter out) { if (StringUtil.isNullOrEmpty(method)) return; if (method.equalsIgnoreCase("getConfig")) {// 获取配置信息 configParser.getConfig(acceptData, request, out); } out.close(); } } src/main/java/com/yeshi/buwan/controller/parser/ClassParser.java
@@ -5,7 +5,6 @@ import com.yeshi.buwan.domain.ad.CommonAdPositionAd; import com.yeshi.buwan.domain.recommend.CategoryRecommendVideo; import com.yeshi.buwan.domain.recommend.HomeRecommendSpecial; import com.yeshi.buwan.domain.solr.SolrAlbumVideo; import com.yeshi.buwan.domain.special.Special; import com.yeshi.buwan.domain.system.DetailSystem; import com.yeshi.buwan.domain.system.DetailSystemConfig; @@ -15,14 +14,11 @@ import com.yeshi.buwan.service.imp.*; import com.yeshi.buwan.service.imp.recommend.CategoryRecommendVideoService; import com.yeshi.buwan.service.inter.recommend.HomeRecommendSpecialService; import com.yeshi.buwan.service.inter.system.SystemConfigService; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.SolrCommonVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.util.*; import com.yeshi.buwan.util.JuHe.VideoResourceUtil; import com.yeshi.buwan.util.ad.CommonAdUtil; import com.yeshi.buwan.util.annotation.RequireUid; import com.yeshi.buwan.util.factory.VideoInfoFactory; import com.yeshi.buwan.util.video.VideoConstant; import com.yeshi.buwan.vo.AcceptData; import com.yeshi.buwan.vo.HomeClassVO; @@ -37,7 +33,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; @Controller public class ClassParser { src/main/java/com/yeshi/buwan/controller/parser/FoundParser.java
@@ -14,7 +14,7 @@ import com.yeshi.buwan.dto.search.SolrVideoSearchFilter; import com.yeshi.buwan.pptv.PPTVUtil; import com.yeshi.buwan.service.imp.*; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.util.Constant; import com.yeshi.buwan.util.factory.VideoInfoFactory; import com.yeshi.buwan.vo.AcceptData; src/main/java/com/yeshi/buwan/controller/parser/SearchParser.java
New file @@ -0,0 +1,123 @@ package com.yeshi.buwan.controller.parser; import com.yeshi.buwan.domain.VideoInfo; import com.yeshi.buwan.domain.solr.SolrAlbumVideo; import com.yeshi.buwan.domain.solr.SolrShortVideo; import com.yeshi.buwan.domain.special.SearchSpecial; import com.yeshi.buwan.domain.video.InternetSearchVideo; import com.yeshi.buwan.dto.search.SolrResultDTO; import com.yeshi.buwan.service.inter.search.SearchSpecialService; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrShortVideoDataManager; import com.yeshi.buwan.util.*; import com.yeshi.buwan.util.factory.VideoInfoFactory; import com.yeshi.buwan.vo.AcceptData; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.springframework.stereotype.Controller; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; @Controller public class SearchParser { @Resource private SearchSpecialService searchSpecialService; @Resource private SolrAlbumVideoDataManager solrAlbumVideoDataManager; @Resource private SolrInternetSearchVideoDataManager solrInternetSearchVideoDataManager; @Resource private SolrShortVideoDataManager solrShortVideoDataManager; /** * 获取专题视频 * * @param acceptData * @param out */ public void getSpecialVideo(AcceptData acceptData, HttpServletRequest request, PrintWriter out) { String key = request.getParameter("key"); SearchSpecial searchSpecial = searchSpecialService.selectByKey(key); if (searchSpecial == null) { out.print(JsonUtil.loadFalseJson("搜索专题不存在")); return; } String page = request.getParameter("page"); if (NumberUtil.isNumeric(page)) { out.print(JsonUtil.loadFalseJson("页码错误")); return; } List<VideoInfo> videoInfoList = new ArrayList<>(); int pageSize = Constant.pageCount; boolean hasNextPage = false; if (searchSpecial.getAlbumQuery() != null) { SolrResultDTO solrResultDTO = solrAlbumVideoDataManager.find(searchSpecial.getAlbumQuery(), Integer.parseInt(page), pageSize); if (solrResultDTO != null) { List<SolrAlbumVideo> list = solrResultDTO.getVideoList(); if (list != null) { for (SolrAlbumVideo av : list) { videoInfoList.add(VideoInfoFactory.create(av)); } if (list.size() > 0) { hasNextPage = true; } } } } if (searchSpecial.getInternetQuery() != null) { SolrResultDTO solrResultDTO = solrInternetSearchVideoDataManager.find(searchSpecial.getInternetQuery(), Integer.parseInt(page), pageSize); if (solrResultDTO != null) { List<InternetSearchVideo> list = solrResultDTO.getVideoList(); if (list != null) { for (InternetSearchVideo av : list) { videoInfoList.add(VideoInfoFactory.create(av)); } if (list.size() > 0) { hasNextPage = true; } } } } if (searchSpecial.getShortQuery() != null) { SolrResultDTO solrResultDTO = solrShortVideoDataManager.find(searchSpecial.getShortQuery(), Integer.parseInt(page), pageSize); if (solrResultDTO != null) { List<SolrShortVideo> list = solrResultDTO.getVideoList(); if (list != null) { for (SolrShortVideo av : list) { videoInfoList.add(VideoInfoFactory.create(av)); } if (list.size() > 0) { hasNextPage = true; } } } } JSONArray array = new JSONArray(); for (int i = 0; i < videoInfoList.size(); i++) { videoInfoList.get(i).setPicture(VideoPictureUtil.getShowPicture(videoInfoList.get(i), acceptData.getPlatform(), acceptData.getVersion() + "")); array.add(StringUtil.outPutResultJson(videoInfoList.get(i))); } JSONObject data = new JSONObject(); data.put("hasNextPage", hasNextPage); data.put("list", array); data.put("column", 2); out.print(JsonUtil.loadTrueAdmin(data)); } } src/main/java/com/yeshi/buwan/controller/parser/UserParser.java
@@ -19,8 +19,8 @@ import com.yeshi.buwan.service.imp.*; import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService; import com.yeshi.buwan.service.inter.system.SystemConfigService; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.util.*; import com.yeshi.buwan.util.JuHe.VideoResourceUtil; import com.yeshi.buwan.util.annotation.RequireUid; @@ -28,22 +28,18 @@ import com.yeshi.buwan.util.factory.VideoInfoFactory; import com.yeshi.buwan.util.log.LoggerUtil; import com.yeshi.buwan.util.log.UserActiveLogFactory; import com.yeshi.buwan.util.rank.IqiyiRankUtil; import com.yeshi.buwan.util.video.VideoCategoryConstant; import com.yeshi.buwan.util.video.VideoConstant; import com.yeshi.buwan.vo.AcceptData; import com.yeshi.buwan.vo.video.VideoListResultVO; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Controller; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpSession; import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Type; import java.util.*; src/main/java/com/yeshi/buwan/dao/search/SearchSpecialDao.java
New file @@ -0,0 +1,98 @@ package com.yeshi.buwan.dao.search; import com.yeshi.buwan.domain.special.SearchSpecial; import com.yeshi.buwan.dao.base.MongodbBaseDao; import com.yeshi.buwan.query.BaseQuery; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.Date; import java.util.List; @Repository public class SearchSpecialDao extends MongodbBaseDao<SearchSpecial> { public void updateSelective(SearchSpecial bean) { Query query = new Query(); Update update = new Update(); query.addCriteria(Criteria.where("id").is(bean.getId())); if (bean.getKey() != null) { update.set("key", bean.getKey()); } if (bean.getName() != null) { update.set("name", bean.getName()); } if (bean.getRemarks() != null) { update.set("remarks", bean.getRemarks()); } if (bean.getColumn() != null) { update.set("column", bean.getColumn()); } if (bean.getSystemId() != null) { update.set("systemId", bean.getSystemId()); } if (bean.getAlbumQuery() != null) { update.set("albumQuery", bean.getAlbumQuery()); } if (bean.getInternetQuery() != null) { update.set("internetQuery", bean.getInternetQuery()); } if (bean.getShortQuery() != null) { update.set("shortQuery", bean.getShortQuery()); } if (bean.getCreateTime() != null) { update.set("createTime", bean.getCreateTime()); } update.set("updateTime", new Date()); update(query, update); } private Query getQuery(DaoQuery daoQuery) { Query query = new Query(); List<Criteria> andList = new ArrayList<>(); if (daoQuery.key != null) { andList.add(Criteria.where("").is(daoQuery.key)); } if (daoQuery.regexName != null) { andList.add(Criteria.where("name").regex(daoQuery.regexName)); } if (daoQuery.systemId != null) { andList.add(Criteria.where("systemId").is(daoQuery.systemId)); } if (andList.size() > 0) { Criteria[] ands = new Criteria[andList.size()]; andList.toArray(ands); query.addCriteria(new Criteria().andOperator(ands)); } return query; } public List<SearchSpecial> list(DaoQuery daoQuery) { Query query = getQuery(daoQuery); query.skip((int) daoQuery.start); query.limit(daoQuery.count); return findList(query); } public long count(DaoQuery daoQuery) { Query query = getQuery(daoQuery); return count(query); } public static class DaoQuery extends BaseQuery { public String regexName; public String key; public String systemId; } } src/main/java/com/yeshi/buwan/domain/solr/SolrShortVideo.java
New file @@ -0,0 +1,166 @@ package com.yeshi.buwan.domain.solr; import com.yeshi.buwan.util.StringUtil; import org.apache.solr.client.solrj.beans.Field; import java.io.Serializable; /** * 搜索引擎中保存的专辑对象 */ public class SolrShortVideo implements Serializable { @Field private String id; @Field(value = "resource_id") private Integer resourceId; @Field private String name; @Field(value = "name_str") private String nameStr; @Field private String picture; //地域 @Field(value = "area") private String area; //标签 @Field(value = "tag") private String tag; //看点 @Field(value = "focus") private String focus; //主分类 @Field(value = "root_video_type") private Integer rootVideoType; //分类名称 @Field(value = "category_names") private String categoryNames; //时长 @Field private String duration; //三方平台更新时间 @Field(value = "third_update_time") private Long thirdUpdateTime; //播放链接 @Field(value = "play_url") private String playUrl; //本地更新时间 @Field(value = "local_update_time") private Long localUpdateTime; public static String createId(Integer resourceId, String playUrl) { return "s-" + resourceId + "-" + StringUtil.Md5(playUrl); } public String getId() { return id; } public void setId(String id) { this.id = id; } public Integer getResourceId() { return resourceId; } public void setResourceId(Integer resourceId) { this.resourceId = resourceId; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getNameStr() { return nameStr; } public void setNameStr(String nameStr) { this.nameStr = nameStr; } public String getArea() { return area; } public void setArea(String area) { this.area = area; } public String getTag() { return tag; } public void setTag(String tag) { this.tag = tag; } public String getFocus() { return focus; } public void setFocus(String focus) { this.focus = focus; } public Integer getRootVideoType() { return rootVideoType; } public void setRootVideoType(Integer rootVideoType) { this.rootVideoType = rootVideoType; } public String getCategoryNames() { return categoryNames; } public void setCategoryNames(String categoryNames) { this.categoryNames = categoryNames; } public String getDuration() { return duration; } public void setDuration(String duration) { this.duration = duration; } public Long getThirdUpdateTime() { return thirdUpdateTime; } public void setThirdUpdateTime(Long thirdUpdateTime) { this.thirdUpdateTime = thirdUpdateTime; } public String getPlayUrl() { return playUrl; } public void setPlayUrl(String playUrl) { this.playUrl = playUrl; } public Long getLocalUpdateTime() { return localUpdateTime; } public void setLocalUpdateTime(Long localUpdateTime) { this.localUpdateTime = localUpdateTime; } public String getPicture() { return picture; } public void setPicture(String picture) { this.picture = picture; } } src/main/java/com/yeshi/buwan/domain/special/SearchSpecial.java
New file @@ -0,0 +1,131 @@ package com.yeshi.buwan.domain.special; import com.yeshi.buwan.dto.search.SolrShortVideoSearchFilter; import com.yeshi.buwan.dto.search.SolrVideoSearchFilter; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.util.Date; /** * 搜索专题 */ @Document(collection = "searchSpecial") public class SearchSpecial { @Id private String id; //名称 private String name; @Indexed private String key; private String systemId; //专辑查询 private SolrVideoSearchFilter albumQuery; //全网搜查询 private SolrVideoSearchFilter internetQuery; //短视频 private SolrShortVideoSearchFilter shortQuery; //备注 private String remarks; //每行多少列 private Integer column; private Date createTime; private Date updateTime; public static String createId(String key, String systemId) { return systemId + "#" + key; } public String getId() { return id; } public void setId(String id) { this.id = id; } public String getKey() { return key; } public void setKey(String key) { this.key = key; } public String getSystemId() { return systemId; } public void setSystemId(String systemId) { this.systemId = systemId; } public SolrVideoSearchFilter getAlbumQuery() { return albumQuery; } public void setAlbumQuery(SolrVideoSearchFilter albumQuery) { this.albumQuery = albumQuery; } public SolrVideoSearchFilter getInternetQuery() { return internetQuery; } public void setInternetQuery(SolrVideoSearchFilter internetQuery) { this.internetQuery = internetQuery; } public Date getCreateTime() { return createTime; } public void setCreateTime(Date createTime) { this.createTime = createTime; } public Date getUpdateTime() { return updateTime; } public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } public SolrShortVideoSearchFilter getShortQuery() { return shortQuery; } public void setShortQuery(SolrShortVideoSearchFilter shortQuery) { this.shortQuery = shortQuery; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getRemarks() { return remarks; } public void setRemarks(String remarks) { this.remarks = remarks; } public Integer getColumn() { return column; } public void setColumn(Integer column) { this.column = column; } } src/main/java/com/yeshi/buwan/dto/search/SolrShortVideoSearchFilter.java
New file @@ -0,0 +1,79 @@ package com.yeshi.buwan.dto.search; /** * 短视频 */ public class SolrShortVideoSearchFilter { private boolean fuzzy; private String key; private String area; private String tag; private Integer rootVideoType; private Integer resourceId; private String categoryName; private String sortKey; public String getKey() { return key; } public void setKey(String key) { this.key = key; } public String getArea() { return area; } public void setArea(String area) { this.area = area; } public String getTag() { return tag; } public void setTag(String tag) { this.tag = tag; } public Integer getRootVideoType() { return rootVideoType; } public void setRootVideoType(Integer rootVideoType) { this.rootVideoType = rootVideoType; } public Integer getResourceId() { return resourceId; } public void setResourceId(Integer resourceId) { this.resourceId = resourceId; } public String getCategoryName() { return categoryName; } public void setCategoryName(String categoryName) { this.categoryName = categoryName; } public String getSortKey() { return sortKey; } public void setSortKey(String sortKey) { this.sortKey = sortKey; } public boolean isFuzzy() { return fuzzy; } public void setFuzzy(boolean fuzzy) { this.fuzzy = fuzzy; } } src/main/java/com/yeshi/buwan/iqiyi/IqiYiNewAPI.java
@@ -467,4 +467,8 @@ } public static void main(String[] args) { IqiyiAlbum2 iqiyiAlbum2 = getAlbumOrVideoDetail(16287138000L); System.out.println(iqiyiAlbum2); } } src/main/java/com/yeshi/buwan/job/SolrJob.java
@@ -2,7 +2,7 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.SolrCommonVideoDataManager; import com.yeshi.buwan.util.SolrUtil; import com.yeshi.buwan.util.StringUtil; src/main/java/com/yeshi/buwan/job/video/VideoUpdateJob.java
@@ -2,7 +2,7 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import org.springframework.stereotype.Component; import javax.annotation.Resource; src/main/java/com/yeshi/buwan/service/imp/SearchService.java
@@ -12,8 +12,8 @@ import com.yeshi.buwan.dto.search.SolrVideoSearchFilter; import com.yeshi.buwan.pptv.PPTVUtil; import com.yeshi.buwan.service.inter.video.VideoInfoExtraService; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.util.*; import com.yeshi.buwan.util.factory.VideoInfoFactory; import com.yeshi.buwan.util.video.VideoConstant; src/main/java/com/yeshi/buwan/service/imp/search/SearchSpecialServiceImpl.java
New file @@ -0,0 +1,91 @@ package com.yeshi.buwan.service.imp.search; import com.yeshi.buwan.dao.search.SearchSpecialDao; import com.yeshi.buwan.domain.special.SearchSpecial; import com.yeshi.buwan.service.inter.search.SearchSpecialService; import com.yeshi.buwan.util.StringUtil; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.Date; import java.util.List; @Service public class SearchSpecialServiceImpl implements SearchSpecialService { @Resource private SearchSpecialDao searchSpecialDao; @Override public SearchSpecial selectByKey(String key) { SearchSpecialDao.DaoQuery daoQuery = new SearchSpecialDao.DaoQuery(); daoQuery.key = key; daoQuery.count = 1; List<SearchSpecial> list = searchSpecialDao.list(daoQuery); if (list != null && list.size() > 0) { return list.get(0); } return null; } @Override public SearchSpecial selectByPrimaryKey(String id) { return searchSpecialDao.get(id); } @Override public List<SearchSpecial> listSpecial(String key, String systemId, int page, int pageSize) { SearchSpecialDao.DaoQuery daoQuery = new SearchSpecialDao.DaoQuery(); daoQuery.regexName = key; daoQuery.systemId = systemId; daoQuery.count = pageSize; daoQuery.start = (page - 1) * pageSize; return searchSpecialDao.list(daoQuery); } @Override public long countSpecial(String key, String systemId) { SearchSpecialDao.DaoQuery daoQuery = new SearchSpecialDao.DaoQuery(); daoQuery.regexName = key; daoQuery.systemId = systemId; return searchSpecialDao.count(daoQuery); } @Override public void update(SearchSpecial special) { if (special.getUpdateTime() == null) { special.setUpdateTime(new Date()); } searchSpecialDao.updateSelective(special); } @Override public void save(SearchSpecial special) throws Exception { if (special == null || StringUtil.isNullOrEmpty(special.getKey())) { throw new Exception("参数不完整"); } if (special.getAlbumQuery() == null && special.getInternetQuery() == null && special.getShortQuery() == null) { throw new Exception("搜索条件不能都为空"); } if (special.getCreateTime() == null) { special.setCreateTime(new Date()); } if (special.getId() == null) { special.setId(SearchSpecial.createId(special.getKey(), special.getSystemId())); } if (special.getUpdateTime() == null) { special.setUpdateTime(new Date()); } searchSpecialDao.save(special); } @Override public void deleteByPrimaryKey(String id) { searchSpecialDao.deleteByPrimaryKey(id); } } src/main/java/com/yeshi/buwan/service/inter/search/SearchSpecialService.java
New file @@ -0,0 +1,56 @@ package com.yeshi.buwan.service.inter.search; import com.yeshi.buwan.domain.special.SearchSpecial; import java.util.List; public interface SearchSpecialService { /** * 根据key查询 * * @param key * @return */ public SearchSpecial selectByKey(String key); /** * 主键查询 * * @param id * @return */ public SearchSpecial selectByPrimaryKey(String id); /** * 列表查询 * * @param key * @param systemId * @param page * @param pageSize * @return */ public List<SearchSpecial> listSpecial(String key, String systemId, int page, int pageSize); public long countSpecial(String key, String systemId); /** * 更新 * * @param special */ public void update(SearchSpecial special); public void save(SearchSpecial special) throws Exception; /** * 删除 * * @param id */ public void deleteByPrimaryKey(String id); } src/main/java/com/yeshi/buwan/service/manager/search/SolrAlbumVideoDataManager.java
File was renamed from src/main/java/com/yeshi/buwan/service/manager/SolrAlbumVideoDataManager.java @@ -1,4 +1,4 @@ package com.yeshi.buwan.service.manager; package com.yeshi.buwan.service.manager.search; import com.yeshi.buwan.domain.ResourceVideo; import com.yeshi.buwan.domain.solr.SolrAlbumVideo; @@ -10,7 +10,6 @@ import com.yeshi.buwan.service.imp.ResourceVideoService; import com.yeshi.buwan.service.imp.VideoInfoService; import com.yeshi.buwan.service.imp.VideoResourceService; import com.yeshi.buwan.service.inter.VideoResourceMapExtraInfoService; import com.yeshi.buwan.service.inter.juhe.AlbumVideoMapService; import com.yeshi.buwan.service.inter.video.VideoInfoExtraService; import com.yeshi.buwan.util.StringUtil; src/main/java/com/yeshi/buwan/service/manager/search/SolrInternetSearchVideoDataManager.java
File was renamed from src/main/java/com/yeshi/buwan/service/manager/SolrInternetSearchVideoDataManager.java @@ -1,4 +1,4 @@ package com.yeshi.buwan.service.manager; package com.yeshi.buwan.service.manager.search; import com.yeshi.buwan.domain.video.InternetSearchVideo; import com.yeshi.buwan.dto.search.SolrResultDTO; src/main/java/com/yeshi/buwan/service/manager/search/SolrShortVideoDataManager.java
New file @@ -0,0 +1,135 @@ package com.yeshi.buwan.service.manager.search; import com.yeshi.buwan.domain.solr.SolrShortVideo; import com.yeshi.buwan.dto.search.SolrResultDTO; import com.yeshi.buwan.dto.search.SolrShortVideoSearchFilter; import com.yeshi.buwan.util.StringUtil; import org.apache.solr.client.solrj.response.UpdateResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.data.domain.Sort; import org.springframework.data.solr.core.SolrTemplate; import org.springframework.data.solr.core.query.Criteria; import org.springframework.data.solr.core.query.Query; import org.springframework.data.solr.core.query.SimpleFilterQuery; import org.springframework.data.solr.core.query.SimpleQuery; import org.springframework.data.solr.core.query.result.ScoredPage; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.util.List; /** * 搜索引擎-短视频数据管理 */ @Component public class SolrShortVideoDataManager { private final String CORE_NAME = "buwan_short_video"; private final Logger logger = LoggerFactory.getLogger(SolrShortVideoDataManager.class); @Resource private SolrTemplate solrTemplate; public void saveOrUpdate(SolrShortVideo solrShortVideo) { if (solrShortVideo.getLocalUpdateTime() == null) { solrShortVideo.setLocalUpdateTime(System.currentTimeMillis()); } UpdateResponse updateResponse = solrTemplate.saveBean(CORE_NAME, solrShortVideo); if (updateResponse.getStatus() == 0) { solrTemplate.commit(CORE_NAME); } else { solrTemplate.rollback(CORE_NAME); } } public void saveOrUpdate(List<SolrShortVideo> shortVideoList) { UpdateResponse updateResponse = solrTemplate.saveBeans(CORE_NAME, shortVideoList); if (updateResponse.getStatus() == 0) { solrTemplate.commit(CORE_NAME); } else { solrTemplate.rollback(CORE_NAME); } } /** * 删除所有 */ // public void clear() { // Query query = new SimpleQuery("name:*"); // UpdateResponse updateResponse = solrTemplate.delete(query); // if (updateResponse.getStatus() == 0) { // solrTemplate.commit(); // } else { // solrTemplate.rollback(); // } // } public void deleteById(String id) { UpdateResponse updateResponse = solrTemplate.deleteById(CORE_NAME, id); if (updateResponse.getStatus() == 0) { solrTemplate.commit(CORE_NAME); } else { solrTemplate.rollback(CORE_NAME); } } public SolrShortVideo findOne(String id) { SolrShortVideo solrVideo = solrTemplate.getById(CORE_NAME, id, SolrShortVideo.class); return solrVideo; } public SolrResultDTO find(SolrShortVideoSearchFilter filter, int page, int pageSize) { if (filter.getKey() == null) { filter.setKey(""); } logger.info(filter.getKey() + "#" + page); Query query = null; if (!StringUtil.isNullOrEmpty(filter.getKey())) { //solr精准检索需要带引号 if (filter.isFuzzy()) { query = new SimpleQuery(new Criteria("name").expression(filter.getKey())); } else { try { query = new SimpleQuery(new Criteria("name_str").startsWith(filter.getKey())); } catch (InvalidDataAccessApiUsageException e) { query = new SimpleQuery(new Criteria("name_str").expression("\"" + filter.getKey() + "\"")); } } } else { query = new SimpleQuery("*:*"); } if (filter.getRootVideoType() != null) { query.addFilterQuery(new SimpleFilterQuery(Criteria.where("root_video_type").is(filter.getRootVideoType()))); } if (filter.getResourceId() != null) { query.addFilterQuery(new SimpleFilterQuery(Criteria.where("resource_id").is(filter.getResourceId()))); } if (filter.getCategoryName() != null) { query.addFilterQuery(new SimpleFilterQuery(Criteria.where("free_type").contains(filter.getCategoryName()))); } if (filter.getArea() != null) { query.addFilterQuery(new SimpleFilterQuery(Criteria.where("area").contains(filter.getArea()))); } /** 设置分页开始记录数(第一页) 默认0 */ query.setOffset((page - 1) * pageSize); /** 设置每页显示记录数,默认10 */ query.setRows(pageSize); if (!StringUtil.isNullOrEmpty(filter.getSortKey())) { query.addSort(new Sort(Sort.Direction.DESC, filter.getSortKey())); } ScoredPage<SolrShortVideo> result = solrTemplate.queryForPage(CORE_NAME, query, SolrShortVideo.class); System.out.println("总记录数:" + result.getTotalElements()); List<SolrShortVideo> list = result.getContent(); return new SolrResultDTO(list, (int) result.getTotalElements()); } } src/main/java/com/yeshi/buwan/util/SpringContext.java
@@ -20,8 +20,8 @@ import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService; import com.yeshi.buwan.service.inter.juhe.Iqiyi2Service; import com.yeshi.buwan.service.inter.juhe.PPTVService; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.util.mq.CMQManager; import io.seata.rm.datasource.DataSourceProxy; import org.slf4j.Logger; src/main/java/com/yeshi/buwan/util/factory/VideoInfoFactory.java
@@ -3,8 +3,10 @@ import com.yeshi.buwan.domain.solr.SolrAlbumVideo; import com.yeshi.buwan.domain.VideoInfo; import com.yeshi.buwan.domain.VideoType; import com.yeshi.buwan.domain.solr.SolrShortVideo; import com.yeshi.buwan.domain.video.InternetSearchVideo; import com.yeshi.buwan.pptv.PPTVUtil; import com.yeshi.buwan.util.TimeUtil; public class VideoInfoFactory { @@ -78,4 +80,36 @@ return video; } public static VideoInfo create(SolrShortVideo solrVideo) { VideoInfo video = new VideoInfo(); video.setMainActor(""); video.setShow(1 + ""); video.setLatestHpicture(""); video.setDuration("0"); video.setScore("9.0"); video.setVpicture(""); video.setLatestVpicture(""); video.setId(solrVideo.getId()); video.setTag(solrVideo.getTag()); video.setHpicture(solrVideo.getPicture()); video.setArea(solrVideo.getArea()); video.setCreatetime(0L); video.setWatchCount("0"); video.setDirector(""); video.setPicture(solrVideo.getPicture()); video.setContentType(1); video.setCommentCount(0); video.setVideocount(1); video.setName(solrVideo.getName()); if (solrVideo.getLocalUpdateTime() != null) video.setUpdatetime(solrVideo.getLocalUpdateTime() + ""); video.setVideoType(new VideoType(solrVideo.getRootVideoType())); video.setDefinition(0); video.setYear(TimeUtil.getGernalTime(solrVideo.getThirdUpdateTime(), "yyyy")); return video; } } src/main/java/com/yeshi/buwan/util/video/shortvideo/IqiyiWebUtil.java
New file @@ -0,0 +1,137 @@ package com.yeshi.buwan.util.video.shortvideo; import com.yeshi.buwan.util.StringUtil; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.yeshi.utils.HttpUtil; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public class IqiyiWebUtil { /** * 网页列表中的请求参数 */ private static class IqiyiWebVideoListParams { private int channelId; private int dataType; private int mode; private String threeCategoryId; public int getChannelId() { return channelId; } public void setChannelId(int channelId) { this.channelId = channelId; } public int getDataType() { return dataType; } public void setDataType(int dataType) { this.dataType = dataType; } public int getMode() { return mode; } public void setMode(int mode) { this.mode = mode; } public String getThreeCategoryId() { return threeCategoryId; } public void setThreeCategoryId(String threeCategoryId) { this.threeCategoryId = threeCategoryId; } } public static IqiyiWebVideoListParams parseParams(String url) { //清除参数 if (url.indexOf("?") > -1) { url = url.substring(0, url.indexOf("?")); } url = url.replace("https://", "").replace("http://", ""); String[] urls = url.split("/"); String host = urls[0]; if (!host.contains(".iqiyi.com")) { return null; } IqiyiWebVideoListParams params = new IqiyiWebVideoListParams(); List<String> threeCategoryIds = new ArrayList<>(); String channelId = urls[2]; String otherParams = urls[3]; String[] sts = otherParams.split("-------------"); if (sts.length > 1) { String threeCategoryId = sts[0]; if (!StringUtil.isNullOrEmpty(threeCategoryId)) { String[] categorys = threeCategoryId.split("-"); for (String cate : categorys) { threeCategoryIds.add(cate + ";must"); } } sts = sts[1].split("-"); String mode = sts[0]; String dataType = sts[2]; params.setMode(Integer.parseInt(mode)); params.setDataType(Integer.parseInt(dataType)); } if (threeCategoryIds.size() > 0) { params.setThreeCategoryId(org.yeshi.utils.StringUtil.concat(threeCategoryIds, ",")); } params.setChannelId(Integer.parseInt(channelId)); return params; } public static List<Long> getVideoTvidList(IqiyiWebVideoListParams params, int page) throws Exception { if (params == null) return null; String url = "https://pcw-api.iqiyi.com/search/recommend/list?"; List<String> paramsList = new ArrayList<>(); paramsList.add("channel_id=" + params.getChannelId()); paramsList.add("data_type=" + params.getDataType()); paramsList.add("mode=" + params.getMode()); paramsList.add("page_id=" + page); paramsList.add("ret_num=48"); if (params.getThreeCategoryId() != null) { paramsList.add("three_category_id=" + URLEncoder.encode(params.getThreeCategoryId())); } url += org.yeshi.utils.StringUtil.concat(paramsList, "&"); Map<String, String> headers = new HashMap<>(); headers.put("referer", "https://list.iqiyi.com/"); headers.put("origin", "https://list.iqiyi.com"); headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"); String result = HttpUtil.get(url, new HashMap<>(), headers); JSONObject resultJSON = JSONObject.fromObject(result); List<Long> tvIds = new ArrayList<>(); if (resultJSON.optString("code").equalsIgnoreCase("A00000")) { JSONObject data = resultJSON.optJSONObject("data"); JSONArray list = data.optJSONArray("list"); for (int i = 0; i < list.size(); i++) { String tvId = list.optJSONObject(i).optString("tvId"); tvIds.add(Long.parseLong(tvId)); } } else throw new Exception(resultJSON.optString("data")); return tvIds; } public static void main(String[] args) throws Exception { List<Long> list = getVideoTvidList(parseParams("https://list.iqiyi.com/www/7/186-------------24-1-2-iqiyi--.html?vfrm=pcw_yule&vfrmblk=C&vfrmrst="), 1); System.out.println(list); } } src/main/java/com/yeshi/buwan/util/video/shortvideo/ShortVideoUtil.java
New file @@ -0,0 +1,18 @@ package com.yeshi.buwan.util.video.shortvideo; public class ShortVideoUtil { /** * 是否为短视频的videoId * * @param id * @return */ public static boolean isShortVideoId(String id) { if (id != null && id.startsWith("s-")) { return true; } return false; } } src/main/java/com/yeshi/buwan/util/video/shortvideo/TencentWebUtil.java
New file @@ -0,0 +1,74 @@ package com.yeshi.buwan.util.video.shortvideo; import com.yeshi.buwan.util.StringUtil; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.yeshi.utils.HttpUtil; import java.net.URLDecoder; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public class TencentWebUtil { public static Map<String, String> parseParams(String url) { //清除参数 String param = null; if (url.indexOf("?") > -1) { param = url.substring(url.indexOf("?")); } if (!url.contains("v.qq.com")) { return null; } String[] params = param.split("&"); Map<String, String> map = new HashMap<>(); for (String p : params) { map.put(p.split("=")[0], URLDecoder.decode(p.split("=")[1])); } return map; } public static void getVideoList(Map<String, String> params, int page) throws Exception { if (params == null) throw new Exception("参数为空"); int pageSize = 30; params.put("append", "1"); params.put("listpage", page + ""); params.put("offset", (page - 1) * pageSize + ""); params.put("pagesize", pageSize + ""); String url = "https://v.qq.com/x/bu/pagesheet/list"; Map<String, String> headers = new HashMap<>(); headers.put("referer", "https://v.qq.com/channel/ent"); headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"); String result = HttpUtil.get(url, params, headers); Document document = Jsoup.parse(result); Elements els = document.getElementsByClass("list_item"); for (int i = 0; i < els.size(); i++) { Element ele = els.get(i); String href = ele.getElementsByTag("a").get(0).attr("href"); String id = ele.getElementsByTag("a").get(0).attr("data-float"); String title = ele.getElementsByTag("a").get(0).attr("title"); title = new String(title.getBytes("ISO-8859-1"), "UTF-8"); String picture = ele.getElementsByTag("img").get(0).attr("src"); picture = picture.startsWith("http") ? picture : "https:" + picture; String duration = ele.getElementsByClass("figure_caption").get(0).ownText(); System.out.println(duration); } } public static void main(String[] args) throws Exception { getVideoList(parseParams("https://v.qq.com/channel/ent?_all=1&channel=ent&iarea=2&itype=-1&listpage=1&sort=40"), 1); } } src/main/resources/env-dev/logback.xml
@@ -137,7 +137,7 @@ </logger> <!--视频搜索关键词 --> <logger name="com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager" level="INFO" additivity="true"> <logger name="com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager" level="INFO" additivity="true"> <appender-ref ref="searchAppender"></appender-ref> </logger> src/main/resources/env-pro/logback.xml
@@ -199,7 +199,7 @@ </logger> <!--视频搜索关键词 --> <logger name="com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager" level="INFO" additivity="true"> <logger name="com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager" level="INFO" additivity="true"> <appender-ref ref="Appender"></appender-ref> </logger> src/test/java/com/hxh/spring/test/GeneralTest.java
@@ -6,6 +6,7 @@ import com.yeshi.buwan.domain.goldcorn.GoldCornTaskActionDetail; import com.yeshi.buwan.domain.live.*; import com.yeshi.buwan.domain.pptv.PPTVPlayProcessRecord; import com.yeshi.buwan.domain.special.SearchSpecial; import com.yeshi.buwan.domain.tb.YouZhiHaoHuoGoods; import com.yeshi.buwan.domain.video.VideoResourceVersionMap; import com.yeshi.buwan.domain.vip.VideoBuyRecord; @@ -24,7 +25,7 @@ @org.junit.Test public void createDao() { try { SpringComponentGenerater.createMongoDao(new MongoDBDaoData.Builder().setBaseDaoClass(MongodbBaseDao.class).setDaoPackageName("com.yeshi.buwan.dao.live.migu").setEntityClass(MiGuLiveListInfo.class).create(), "D:\\workspace\\BuWan\\src\\main\\java\\com\\yeshi\\buwan\\dao\\live\\migu\\"); SpringComponentGenerater.createMongoDao(new MongoDBDaoData.Builder().setBaseDaoClass(MongodbBaseDao.class).setDaoPackageName("com.yeshi.buwan.dao.search").setEntityClass(SearchSpecial.class).create(), "D:\\workspace\\BuWan\\src\\main\\java\\com\\yeshi\\buwan\\dao\\search\\"); } catch (Exception e) { src/test/java/com/hxh/spring/test/Iqiyi2.java
@@ -17,8 +17,10 @@ import com.yeshi.buwan.service.imp.ResourceVideoService; import com.yeshi.buwan.service.imp.VideoInfoService; import com.yeshi.buwan.service.inter.juhe.Iqiyi2Service; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.util.*; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.util.HtmlToolUtil; import com.yeshi.buwan.util.StringUtil; import com.yeshi.buwan.util.TimeUtil; import com.yeshi.buwan.util.mq.CMQManager; import net.sf.json.JSONArray; import org.junit.Test; src/test/java/com/hxh/spring/test/SolrTest.java
@@ -8,14 +8,12 @@ import com.yeshi.buwan.domain.video.InternetSearchVideo; import com.yeshi.buwan.dto.search.SolrResultDTO; import com.yeshi.buwan.dto.search.SolrVideoSearchFilter; import com.yeshi.buwan.pptv.PPTVUtil; import com.yeshi.buwan.service.imp.VideoInfoService; import com.yeshi.buwan.service.inter.juhe.AlbumVideoMapService; import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService; import com.yeshi.buwan.service.manager.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrAlbumVideoDataManager; import com.yeshi.buwan.service.manager.SolrCommonVideoDataManager; import com.yeshi.buwan.service.manager.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.util.Constant; import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.util.StringUtil; import com.yeshi.buwan.util.factory.VideoInfoFactory; import com.yeshi.buwan.util.video.VideoConstant; src/test/java/com/hxh/spring/test/video/BilibiliTest.java
@@ -1,12 +1,9 @@ package com.hxh.spring.test.video; import com.yeshi.buwan.job.video.MogoTVVideoUpdate; import com.yeshi.buwan.mogotv.entity.MogoTVClipInfo; import com.yeshi.buwan.service.inter.juhe.BilibiliVideoService; import com.yeshi.buwan.service.inter.juhe.InternetSearchVideoService; import com.yeshi.buwan.service.inter.juhe.MogoTVService; import com.yeshi.buwan.service.manager.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.service.manager.search.SolrInternetSearchVideoDataManager; import com.yeshi.buwan.videos.bilibili.BilibiliApiUtil; import com.yeshi.buwan.videos.bilibili.BilibiliUtil; import com.yeshi.buwan.videos.bilibili.entity.BilibiliMediaInfo; @@ -17,7 +14,6 @@ import org.springframework.test.context.web.WebAppConfiguration; import javax.annotation.Resource; import java.util.List; // @RunWith(SpringJUnit4ClassRunner.class) //使用junit4进行测试