src/main/java/com/yeshi/buwan/dao/juhe/youku/YouKuShowDetailDao.java
New file @@ -0,0 +1,113 @@ package com.yeshi.buwan.dao.juhe.youku; import com.yeshi.buwan.youku.entity.YouKuShowDetail; import com.yeshi.buwan.dao.base.MongodbBaseDao; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Repository; import java.util.Date; @Repository public class YouKuShowDetailDao extends MongodbBaseDao<YouKuShowDetail>{ public void updateSelective(YouKuShowDetail bean) { Query query = new Query(); Update update=new Update(); query.addCriteria(Criteria.where("id").is(bean.getId())); if(bean.getState() != null) { update.set("state", bean.getState()); } if(bean.getCopyright_status() != null) { update.set("copyright_status", bean.getCopyright_status()); } if(bean.getName() != null) { update.set("name", bean.getName()); } if(bean.getSubtitle() != null) { update.set("subtitle", bean.getSubtitle()); } if(bean.getAlias() != null) { update.set("alias", bean.getAlias()); } if(bean.getLink() != null) { update.set("link", bean.getLink()); } if(bean.getPlay_link() != null) { update.set("play_link", bean.getPlay_link()); } if(bean.getPoster() != null) { update.set("poster", bean.getPoster()); } if(bean.getPoster_large() != null) { update.set("poster_large", bean.getPoster_large()); } if(bean.getThumbnail() != null) { update.set("thumbnail", bean.getThumbnail()); } if(bean.getThumbnail_large() != null) { update.set("thumbnail_large", bean.getThumbnail_large()); } if(bean.getGenre() != null) { update.set("genre", bean.getGenre()); } if(bean.getArea() != null) { update.set("area", bean.getArea()); } if(bean.getCompleted() != null) { update.set("completed", bean.getCompleted()); } if(bean.getEpisode_count() != null) { update.set("episode_count", bean.getEpisode_count()); } if(bean.getEpisode_collected() != null) { update.set("episode_collected", bean.getEpisode_collected()); } if(bean.getEpisode_updated() != null) { update.set("episode_updated", bean.getEpisode_updated()); } if(bean.getUpdate_notice() != null) { update.set("update_notice", bean.getUpdate_notice()); } if(bean.getScore() != null) { update.set("score", bean.getScore()); } if(bean.getPaid() != null) { update.set("paid", bean.getPaid()); } if(bean.getPublished() != null) { update.set("published", bean.getPublished()); } if(bean.getReleased() != null) { update.set("released", bean.getReleased()); } if(bean.getReleasedate_mainland() != null) { update.set("releasedate_mainland", bean.getReleasedate_mainland()); } if(bean.getCreatetime() != null) { update.set("createtime", bean.getCreatetime()); } if(bean.getCategory() != null) { update.set("category", bean.getCategory()); } if(bean.getDescription() != null) { update.set("description", bean.getDescription()); } if(bean.getRank() != null) { update.set("rank", bean.getRank()); } if(bean.getDouban_num() != null) { update.set("douban_num", bean.getDouban_num()); } if(bean.getAttr() != null) { update.set("attr", bean.getAttr()); } if(bean.getPremium() != null) { update.set("premium", bean.getPremium()); } update.set("updateTime", new Date()); update(query, update); } } src/main/java/com/yeshi/buwan/dao/juhe/youku/YouKuVideoDao.java
New file @@ -0,0 +1,61 @@ package com.yeshi.buwan.dao.juhe.youku; import com.yeshi.buwan.youku.entity.YouKuVideo; import com.yeshi.buwan.dao.base.MongodbBaseDao; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Repository; import java.util.Date; @Repository public class YouKuVideoDao extends MongodbBaseDao<YouKuVideo>{ public void updateSelective(YouKuVideo bean) { Query query = new Query(); Update update=new Update(); query.addCriteria(Criteria.where("videoid").is(bean.getVideoid())); if(bean.getThumbnail() != null) { update.set("thumbnail", bean.getThumbnail()); } if(bean.getLink() != null) { update.set("link", bean.getLink()); } if(bean.getShowId() != null) { update.set("showId", bean.getShowId()); } if(bean.getTitle() != null) { update.set("title", bean.getTitle()); } if(bean.getSeconds() != null) { update.set("seconds", bean.getSeconds()); } if(bean.getShow_videostage() != null) { update.set("show_videostage", bean.getShow_videostage()); } if(bean.getThumbnail_v2() != null) { update.set("thumbnail_v2", bean.getThumbnail_v2()); } if(bean.getRc_title() != null) { update.set("rc_title", bean.getRc_title()); } if(bean.getPublishtime() != null) { update.set("publishtime", bean.getPublishtime()); } if(bean.getDownload_status() != null) { update.set("download_status", bean.getDownload_status()); } if(bean.getState() != null) { update.set("state", bean.getState()); } if(bean.getCategory() != null) { update.set("category", bean.getCategory()); } if(bean.getIs_panorama() != null) { update.set("is_panorama", bean.getIs_panorama()); } update.set("updateTime", new Date()); update(query, update); } } src/main/java/com/yeshi/buwan/job/video/YouKuVideoUpdate.java
New file @@ -0,0 +1,97 @@ package com.yeshi.buwan.job.video; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import com.yeshi.buwan.dao.juhe.pptv.PPTVSeriesDao; import com.yeshi.buwan.dao.juhe.pptv.VideoPPTVMapDao; import com.yeshi.buwan.pptv.PPTVApiUtil; import com.yeshi.buwan.pptv.entity.PPTVSeries; import com.yeshi.buwan.service.inter.juhe.PPTVService; import com.yeshi.buwan.service.inter.juhe.YouKuService; import com.yeshi.buwan.util.StringUtil; import com.yeshi.buwan.youku.YouKuApiUtil; import com.yeshi.buwan.youku.entity.YouKuShowDetail; import com.yeshi.buwan.youku.entity.YouKuShowSimple; import com.yeshi.buwan.youku.entity.YouKuVideo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.util.List; @Component public class YouKuVideoUpdate { private final static Logger logger = LoggerFactory.getLogger(YouKuVideoUpdate.class); @Resource private YouKuService youKuService; private YouKuShowDetail getShowDetail(String showId) throws Exception { YouKuShowDetail detail = YouKuApiUtil.getShowDetail(showId); int pageSize = 100; YouKuApiUtil.ListResultDTO resultDTO = YouKuApiUtil.getVideoList(showId, 1, pageSize); if (resultDTO == null) throw new Exception("视频列表获取出错"); List<YouKuVideo> videoList = resultDTO.getList(); if (videoList.size() < resultDTO.getTotal()) { int totalPage = resultDTO.getTotal() % pageSize == 0 ? resultDTO.getTotal() / pageSize : resultDTO.getTotal() / pageSize + 1; for (int i = 1; i < totalPage; i++) { resultDTO = YouKuApiUtil.getVideoList(showId, i + 1, pageSize); if (resultDTO != null) videoList.addAll(resultDTO.getList()); } } detail.setVideoList(videoList); return detail; } private void updateCategory(String categoryName) throws Exception { YouKuApiUtil.ListResultDTO dto = YouKuApiUtil.getShowListByCategory(categoryName, 1, 20); if (dto != null) { int pageSize = 100; int totalPage = dto.getTotal() % pageSize == 0 ? dto.getTotal() / pageSize : dto.getTotal() / pageSize + 1; totalPage = totalPage > 20 ? 20 : totalPage; for (int i = 0; i < totalPage; i++) { dto = YouKuApiUtil.getShowListByCategory(categoryName, i + 1, pageSize); for (YouKuShowSimple simple : (List<YouKuShowSimple>) dto.getList()) { System.out.println(simple.getName() + ":" + simple.getId()); try { YouKuShowDetail detail = getShowDetail(simple.getId()); youKuService.save(detail); } catch (Exception e) { e.printStackTrace(); } } } } } /** * 更新最近几天的视频 * * @param param * @return * @throws Exception */ @XxlJob("video-update-youku-updateVideo") public ReturnT<String> updateLatestVideo(String param) throws Exception { String[] types = new String[]{ "电影", "电视剧", "动漫", "综艺" }; if (!StringUtil.isNullOrEmpty(param)) { updateCategory(param); } else { for (String type : types) { updateCategory(type); } } return ReturnT.SUCCESS; } } src/main/java/com/yeshi/buwan/service/imp/juhe/YouKuServiceImpl.java
New file @@ -0,0 +1,34 @@ package com.yeshi.buwan.service.imp.juhe; import com.yeshi.buwan.dao.juhe.youku.YouKuShowDetailDao; import com.yeshi.buwan.dao.juhe.youku.YouKuVideoDao; import com.yeshi.buwan.service.inter.juhe.YouKuService; import com.yeshi.buwan.youku.entity.YouKuShowDetail; import com.yeshi.buwan.youku.entity.YouKuVideo; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.Date; @Service public class YouKuServiceImpl implements YouKuService { @Resource private YouKuShowDetailDao youKuShowDetailDao; @Resource private YouKuVideoDao youKuVideoDao; @Override public void save(YouKuShowDetail detail) { if (detail.getUpdateTime() == null) detail.setUpdateTime(new Date()); youKuShowDetailDao.save(detail); if (detail.getVideoList() != null) for (YouKuVideo video : detail.getVideoList()) { if (video.getUpdateTime() == null) { video.setUpdateTime(new Date()); } youKuVideoDao.save(video); } } } src/main/java/com/yeshi/buwan/service/inter/juhe/YouKuService.java
New file @@ -0,0 +1,13 @@ package com.yeshi.buwan.service.inter.juhe; import com.yeshi.buwan.youku.entity.YouKuShowDetail; public interface YouKuService { /** * 保存 * @param detail */ public void save(YouKuShowDetail detail); } src/main/java/com/yeshi/buwan/tencent/TencentVideoUtil.java
New file @@ -0,0 +1,186 @@ package com.yeshi.buwan.tencent; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.yeshi.buwan.tencent.entity.TencentCoverInfo; import net.sf.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.yeshi.utils.HttpUtil; import javax.script.Invocable; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.io.IOException; import java.lang.reflect.Type; import java.util.*; public class TencentVideoUtil { static ScriptEngine jsEngine = null; // static { // if (jdGoodsJs == null) // jdGoodsJs = getJDGoodsJS(); // ScriptEngineManager manager = new ScriptEngineManager(); // jsEngine = manager.getEngineByName("javascript"); // try { // jsEngine.eval(jdGoodsJs); // } catch (ScriptException e) { // e.printStackTrace(); // } // // if (jsEngine instanceof Invocable) { // Invocable in = (Invocable) jsEngine; // Object goodsId = in.invokeFunction("getGoodsId", link); // if (goodsId != null) // return goodsId.toString().trim(); // } // // } private static List<String> objToArray(JSONObject root, String key) { if (root.optJSONObject(key) == null) { return null; } Gson gson = new Gson(); List<String> directors = new ArrayList<>(); if (root.optJSONObject(key).isArray()) { Type type = new TypeToken<List<String>>() { }.getType(); directors = gson.fromJson(root.optJSONArray(key).toString(), type); } else { JSONObject director = root.optJSONObject(key); for (Iterator<String> its = director.keys(); its.hasNext(); ) { String p = its.next(); directors.add(director.optString(p)); } } root.remove(key); return directors; } public static TencentCoverInfo getCoverInfo(String url) throws Exception { String script = null; Document doc = Jsoup.connect(url).timeout(10000).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36").get(); Elements els = doc.getElementsByTag("script"); for (int i = 0; i < els.size(); i++) { if (els.get(i).html().indexOf("var COVER_INFO") > -1) { script = els.get(i).html(); break; } } script = script.replace("window.", "_window."); script = " var _window={};" + "var document={getElementsByTagName:function(tag){return [''];}};" + script; script += "\n function getCoverInfo(){return COVER_INFO }; function getVideoInfo(){return VIDEO_INFO}"; ScriptEngineManager manager = new ScriptEngineManager(); jsEngine = manager.getEngineByName("javascript"); try { jsEngine.eval(script); } catch (ScriptException e) { e.printStackTrace(); } Gson gson = new Gson(); if (jsEngine instanceof Invocable) { Invocable in = (Invocable) jsEngine; Object coverInfo = in.invokeFunction("getCoverInfo"); JSONObject root = JSONObject.fromObject(gson.toJson(coverInfo)); System.out.println(root.toString()); List<TencentCoverInfo.VipIdsBean> vipIdsBeans = new ArrayList<>(); if (root.optJSONObject("vip_ids").isArray()) { Type type = new TypeToken<List<TencentCoverInfo.VipIdsBean>>() { }.getType(); vipIdsBeans = gson.fromJson(root.optJSONArray("vip_ids").toString(), type); } else { JSONObject vipIds = root.optJSONObject("vip_ids"); for (Iterator<String> its = vipIds.keys(); its.hasNext(); ) { String p = its.next(); TencentCoverInfo.VipIdsBean idsBean = gson.fromJson(vipIds.optJSONObject(p).toString(), TencentCoverInfo.VipIdsBean.class); idsBean.setP(Integer.parseInt(p)); vipIdsBeans.add(idsBean); } } root.remove("vip_ids"); try { List<String> subTypes = objToArray(root, "subtype"); List<String> directors = objToArray(root, "director"); List<String> leadingActor = objToArray(root, "leading_actor"); List<String> subGenre = objToArray(root, "sub_genre"); TencentCoverInfo coverInfoBean = gson.fromJson(root.toString(), TencentCoverInfo.class); coverInfoBean.setVip_ids(vipIdsBeans); coverInfoBean.setSubtype(subTypes); coverInfoBean.setDirector(directors); coverInfoBean.setLeading_actor(leadingActor); coverInfoBean.setSub_genre(subGenre); return coverInfoBean; } catch (Exception e) { e.printStackTrace(); } // System.out.println(coverInfoBean); } return null; } private static void parseList(String url) { try { Document document = Jsoup.connect(url).timeout(5000).get(); Elements els = document.getElementsByAttributeValue("class", "list_item"); for (int i = 0; i < els.size(); i++) { String href = els.get(i).getElementsByTag("a").get(0).attr("href"); String img = els.get(i).getElementsByTag("img").get(0).attr("src"); img = img.startsWith("http") ? img : "http:" + img; try { TencentCoverInfo coverInfo = getCoverInfo(href); System.out.println(coverInfo); } catch (Exception e) { e.printStackTrace(); } } } catch (IOException e) { e.printStackTrace(); } } public static void main(String[] args) { // try { // TencentCoverInfo info = getCoverInfo("https://v.qq.com/x/cover/mzc00200fw94jce.html"); // // info = getCoverInfo("https://v.qq.com/x/cover/mzc00200s0ii272/r0036xibgw2.html"); // info = getCoverInfo("https://v.qq.com/x/cover/ylgl3m6wo0sypou/w0036x9c5c7.html"); // info = getCoverInfo("https://v.qq.com/x/cover/ylgl3m6wo0sypou/w0036x9c5c7.html"); // // // info = getCoverInfo("https://v.qq.com/x/cover/mzc00200tlv15ub.html"); // System.out.println(info); // } catch (Exception e) { // e.printStackTrace(); // } int page = 1; parseList(String.format("https://v.qq.com/x/bu/pagesheet/list?_all=1&append=1&channel=cartoon&ipay=2&listpage=%s&offset=%s&pagesize=30&sort=18", page, (page - 1) * 30)); } } src/main/java/com/yeshi/buwan/tencent/entity/TencentCoverInfo.java
New file @@ -0,0 +1,333 @@ package com.yeshi.buwan.tencent.entity; import java.util.List; public class TencentCoverInfo { /** * id : mzc00200fw94jce * director : ["宋小宝"] * douban_score : 6.0 * leading_actor : ["宋小宝","马丽","沙溢","张一山"] * vip_ids : [{"F":7,"V":"h0035b23dyt"}] * pay_status : 6 * year : 2021 * episode_updated : null * episode_all : null * payfree_num : null * current_num : null * publish_date : 2021-02-12 * title : 发财日记 * description : 故事讲述上世纪90年代初,穷小子小宝追随同村“发小儿”二哥,一路从贫穷朴实的东北农村来到了经济飞速发展的深圳。期间,两人无意中收留了一个嗷嗷待哺的婴儿,生活困窘的小宝和二哥该如何将孩子抚养长大?当小宝和二哥与同在深圳打拼的创业女青年马露“狭路相逢,三人之间又将发生哪些令人捧腹的爆笑故事?一段笑泪参半、起伏跌宕的小人物奋斗史就此展开 * cover_id : mzc00200fw94jce * type_name : 电影 * area_name : 内地 * horizontal_pic_url : null * vertical_pic_url : null * second_title : 宋小宝马丽拯救不开心 * langue : 普通话 * subtype : ["喜剧","剧情"] * main_genre : 喜剧 * score : {"c_mix_score":"8.4","hot":"9.99151","score":"8.4"} * sub_genre : ["剧情"] */ private String id; private String douban_score; private int pay_status; private String year; private String episode_updated; private String episode_all; private String current_num; private String publish_date; private String title; private String description; private String cover_id; private String type_name; private String area_name; private String horizontal_pic_url; private String vertical_pic_url; private String second_title; private String langue; private String main_genre; private ScoreBean score; private List<String> director; private List<String> leading_actor; private List<VipIdsBean> vip_ids; private List<String> subtype; private List<String> sub_genre; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getDouban_score() { return douban_score; } public void setDouban_score(String douban_score) { this.douban_score = douban_score; } public int getPay_status() { return pay_status; } public void setPay_status(int pay_status) { this.pay_status = pay_status; } public String getYear() { return year; } public void setYear(String year) { this.year = year; } public String getEpisode_updated() { return episode_updated; } public void setEpisode_updated(String episode_updated) { this.episode_updated = episode_updated; } public String getEpisode_all() { return episode_all; } public void setEpisode_all(String episode_all) { this.episode_all = episode_all; } public String getCurrent_num() { return current_num; } public void setCurrent_num(String current_num) { this.current_num = current_num; } public String getPublish_date() { return publish_date; } public void setPublish_date(String publish_date) { this.publish_date = publish_date; } public String getTitle() { return title; } public void setTitle(String title) { this.title = title; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getCover_id() { return cover_id; } public void setCover_id(String cover_id) { this.cover_id = cover_id; } public String getType_name() { return type_name; } public void setType_name(String type_name) { this.type_name = type_name; } public String getArea_name() { return area_name; } public void setArea_name(String area_name) { this.area_name = area_name; } public String getHorizontal_pic_url() { return horizontal_pic_url; } public void setHorizontal_pic_url(String horizontal_pic_url) { this.horizontal_pic_url = horizontal_pic_url; } public String getVertical_pic_url() { return vertical_pic_url; } public void setVertical_pic_url(String vertical_pic_url) { this.vertical_pic_url = vertical_pic_url; } public String getSecond_title() { return second_title; } public void setSecond_title(String second_title) { this.second_title = second_title; } public String getLangue() { return langue; } public void setLangue(String langue) { this.langue = langue; } public String getMain_genre() { return main_genre; } public void setMain_genre(String main_genre) { this.main_genre = main_genre; } public ScoreBean getScore() { return score; } public void setScore(ScoreBean score) { this.score = score; } public List<String> getDirector() { return director; } public void setDirector(List<String> director) { this.director = director; } public List<String> getLeading_actor() { return leading_actor; } public void setLeading_actor(List<String> leading_actor) { this.leading_actor = leading_actor; } public List<VipIdsBean> getVip_ids() { return vip_ids; } public void setVip_ids(List<VipIdsBean> vip_ids) { this.vip_ids = vip_ids; } public List<String> getSubtype() { return subtype; } public void setSubtype(List<String> subtype) { this.subtype = subtype; } public List<String> getSub_genre() { return sub_genre; } public void setSub_genre(List<String> sub_genre) { this.sub_genre = sub_genre; } public static class ScoreBean { /** * c_mix_score : 8.4 * hot : 9.99151 * score : 8.4 */ private String c_mix_score; private String hot; private String score; public String getC_mix_score() { return c_mix_score; } public void setC_mix_score(String c_mix_score) { this.c_mix_score = c_mix_score; } public String getHot() { return hot; } public void setHot(String hot) { this.hot = hot; } public String getScore() { return score; } public void setScore(String score) { this.score = score; } } public static class VipIdsBean { /** * F : 7 * V : h0035b23dyt */ private int F; private String V; private int P;//position private String D;//date 时间 public String getD() { return D; } public void setD(String d) { D = d; } public int getP() { return P; } public void setP(int p) { P = p; } public int getF() { return F; } public void setF(int F) { this.F = F; } public String getV() { return V; } public void setV(String V) { this.V = V; } } } src/main/java/com/yeshi/buwan/youku/YouKuApiUtil.java
New file @@ -0,0 +1,151 @@ package com.yeshi.buwan.youku; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import com.yeshi.buwan.youku.entity.YouKuShowDetail; import com.yeshi.buwan.youku.entity.YouKuShowSimple; import com.yeshi.buwan.youku.entity.YouKuVideo; import org.json.JSONObject; import org.yeshi.utils.HttpUtil; import org.yeshi.utils.StringUtil; import java.io.UnsupportedEncodingException; import java.lang.reflect.Type; import java.net.URLEncoder; import java.util.*; public class YouKuApiUtil { private final static String APP_ID = "758bef946d0050ef"; private final static String APP_SECRET = "e113a131d49e1f837402e2807f2daf58"; private static String request(String action, Map<String, String> ps) { Map<String, String> params = new HashMap<>(); params.put("action", action); params.put("client_id", APP_ID); params.put("format", "json"); params.put("timestamp", System.currentTimeMillis() / 1000 + ""); params.put("version", "3.0"); params.put("format", "json"); params.putAll(ps); List<String> list = new ArrayList<>(); for (Iterator<String> its = params.keySet().iterator(); its.hasNext(); ) { String key = its.next(); try { list.add(key + URLEncoder.encode(params.get(key), "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } } Collections.sort(list); String sign = StringUtil.Md5(StringUtil.concat(list, "") + APP_SECRET); params.put("sign", sign); ps.put("opensysparams", new JSONObject(params).toString()); return HttpUtil.get("https://openapi.youku.com/router/rest.json", ps, new HashMap<>()); } /** * 获取节目详情 * * @param id * @return */ public static YouKuShowDetail getShowDetail(String id) { String url = String.format("https://openapi.youku.com/v2/shows/show.json?client_id=%s&show_id=%s", APP_ID, id); String result = HttpUtil.get(url); return new Gson().fromJson(result, YouKuShowDetail.class); } /** * 根据分类拉取数据 * * @param category * @param page */ public static ListResultDTO getShowListByCategory(String category, int page, int pageSize) { try { String url = String.format("https://openapi.youku.com/v2/shows/by_category.json?client_id=%s&category=%s&page=%s&count=" + pageSize, APP_ID, URLEncoder.encode(category, "UTF-8"), page); String result = HttpUtil.get(url); net.sf.json.JSONObject resultJson = net.sf.json.JSONObject.fromObject(result); int total = resultJson.optInt("total"); Type type = new TypeToken<List<YouKuShowSimple>>() { }.getType(); List<YouKuShowSimple> list = new Gson().fromJson(resultJson.optJSONArray("shows").toString(), type); return new ListResultDTO(list, total); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } return null; } public static ListResultDTO getVideoList(String showId, int page, int pageSize) { Map<String, String> params = new HashMap<>(); params.put("show_id", showId); params.put("show_videotype", "正片"); params.put("page", page + ""); params.put("pageLength", pageSize + ""); String result = YouKuApiUtil.request("youku.api.video.byprogram.get", params); System.out.println(result); net.sf.json.JSONObject resultJson = net.sf.json.JSONObject.fromObject(result); int total = resultJson.optInt("total"); Type type = new TypeToken<List<YouKuVideo>>() { }.getType(); List<YouKuVideo> videoList = new Gson().fromJson(resultJson.optJSONArray("videos").toString(), type); if (videoList != null) for (YouKuVideo video : videoList) { video.setShowId(showId); } return new ListResultDTO(videoList, total); } public static void main(String[] args) { // YouKuShowDetail detail = getShowDetail("1e61efbfbdefbfbd04ef"); // System.out.println(detail); getVideoList("0a17e614d7e311e68ce4", 1, 20); // ListResultDTO resultDTO = getShowListByCategory("电视剧", 1); // System.out.println(resultDTO); } public static class ListResultDTO { private List list; private int total; public ListResultDTO(List list, int total) { this.list = list; this.total = total; } public List getList() { return list; } public void setList(List list) { this.list = list; } public int getTotal() { return total; } public void setTotal(int total) { this.total = total; } } } src/main/java/com/yeshi/buwan/youku/entity/YouKuShowDetail.java
New file @@ -0,0 +1,536 @@ package com.yeshi.buwan.youku.entity; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Transient; import org.springframework.data.mongodb.core.mapping.Document; import java.util.Date; import java.util.List; @Document(collection = "youKuShow") public class YouKuShowDetail { /** * id : 1e61efbfbdefbfbd04ef * state : normal * copyright_status : authorized * name : 司藤 * subtitle : 景甜张彬彬联手隔世追爱 * alias : null * link : http://www.youku.com/show_page/id_z1e61efbfbdefbfbd04ef.html * play_link : http://v.youku.com/v_show/id_XNTAzNDM2MDY5Ng==.html * poster : http://r1.ykimg.com/050D00006045B5A313EA3508551952B5 * poster_large : http://r1.ykimg.com/050E00006045B5A313EA3508551952B5 * thumbnail : http://r1.ykimg.com/050B00006045B5AE13EA350855E2A59A * thumbnail_large : http://r1.ykimg.com/050C00006045B5AE13EA350855E2A59A * genre : 爱情,奇幻 * area : 中国 * completed : 0 * episode_count : 30 * episode_collected : 30 * episode_updated : 22 * update_notice : VIP会员每周日至周二20点更新2集,非会员每周日至周三20点更新1集 * view_count : 536371225 * score : 0.000 * paid : 1 * published : 2021-03-08 * released : 2021-03-08 * createtime : 2018-01-29 13:27:38 * category : 电视剧 * description : 该剧讲述了青年设计师秦放(张彬彬 饰)在一次寻根过程中,因一次意外与沉睡数十年的司藤(景甜 饰)相遇,在帮助司藤寻找身世真相的过程中,两人彼此相知并克服重重阻碍共同成长的故事。 * rank : 0 * douban_num : 27605542 * attr : {"director":[{"id":"387687","name":"李木戈","link":"http://www.youku.com/star_page/uid_UMTU1MDc0OA==.html"}],"performer":[{"id":"295693","name":"景甜","character":"司藤","link":"http://www.youku.com/star_page/uid_UMTE4Mjc3Mg==.html"},{"id":"835783","name":"张彬彬","character":"秦放","link":"http://www.youku.com/star_page/uid_UMzM0MzEzMg==.html"},{"id":"945700","name":"李沐宸","character":"沈银灯","link":"http://www.youku.com/star_page/uid_UMzc4MjgwMA==.html"},{"id":"876070","name":"张亦驰","character":"颜福瑞","link":"http://www.youku.com/star_page/uid_UMzUwNDI4MA==.html"},{"id":"374580","name":"吴俊余","character":"王乾坤","link":"http://www.youku.com/star_page/uid_UMTQ5ODMyMA==.html"},{"id":"852947","name":"金泽灏","character":"单志刚","link":"http://www.youku.com/star_page/uid_UMzQxMTc4OA==.html"},{"id":"966916","name":"潘一祎","character":"安蔓","link":"http://www.youku.com/star_page/uid_UMzg2NzY2NA==.html"},{"id":"242924","name":"张定涵","character":"贾桂芝","link":"http://www.youku.com/star_page/uid_UOTcxNjk2.html"},{"id":"23259","name":"吕行","character":"赵江龙","link":"http://www.youku.com/star_page/uid_UOTMwMzY=.html"},{"id":"884664","name":"王冠","character":"周万东","link":"http://www.youku.com/star_page/uid_UMzUzODY1Ng==.html"},{"id":"264767","name":"师悦玲","character":"孔菁华","link":"http://www.youku.com/star_page/uid_UMTA1OTA2OA==.html"},{"id":"12766","name":"邵峰","character":"丘山","link":"http://www.youku.com/star_page/uid_UNTEwNjQ=.html"},{"id":"221860","name":"李依晓","character":"苅族","link":"http://www.youku.com/star_page/uid_UODg3NDQw.html"},{"id":"24067","name":"袁成杰","character":"邵琰宽","link":"http://www.youku.com/star_page/uid_UOTYyNjg=.html"}]} * premium : {"onlinetime":"2021-03-07 00:00:00","permit_duration":"2","price":"0"} */ @Id private String id; private String state; private String copyright_status; private String name;//节目名称 private String subtitle; private Object alias; private String link; private String play_link; private String poster; private String poster_large; private String thumbnail; private String thumbnail_large; private String genre; private String area; private Integer completed; private String episode_count; private String episode_collected; private String episode_updated; private String update_notice; private String score; private Integer paid; private String published; private String released; private String releasedate_mainland; private String createtime; private String category; private String description; private Integer rank; private String douban_num; private AttrBean attr; private PremiumBean premium; private Date updateTime; public Date getUpdateTime() { return updateTime; } public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } @Transient private List<YouKuVideo> videoList; public List<YouKuVideo> getVideoList() { return videoList; } public void setVideoList(List<YouKuVideo> videoList) { this.videoList = videoList; } public String getId() { return id; } public void setId(String id) { this.id = id; } public String getState() { return state; } public void setState(String state) { this.state = state; } public String getCopyright_status() { return copyright_status; } public void setCopyright_status(String copyright_status) { this.copyright_status = copyright_status; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getSubtitle() { return subtitle; } public void setSubtitle(String subtitle) { this.subtitle = subtitle; } public Object getAlias() { return alias; } public void setAlias(Object alias) { this.alias = alias; } public String getLink() { return link; } public void setLink(String link) { this.link = link; } public String getPlay_link() { return play_link; } public void setPlay_link(String play_link) { this.play_link = play_link; } public String getPoster() { return poster; } public void setPoster(String poster) { this.poster = poster; } public String getPoster_large() { return poster_large; } public void setPoster_large(String poster_large) { this.poster_large = poster_large; } public String getThumbnail() { return thumbnail; } public void setThumbnail(String thumbnail) { this.thumbnail = thumbnail; } public String getThumbnail_large() { return thumbnail_large; } public void setThumbnail_large(String thumbnail_large) { this.thumbnail_large = thumbnail_large; } public String getGenre() { return genre; } public void setGenre(String genre) { this.genre = genre; } public String getArea() { return area; } public void setArea(String area) { this.area = area; } public Integer getCompleted() { return completed; } public void setCompleted(Integer completed) { this.completed = completed; } public String getEpisode_count() { return episode_count; } public void setEpisode_count(String episode_count) { this.episode_count = episode_count; } public String getEpisode_collected() { return episode_collected; } public void setEpisode_collected(String episode_collected) { this.episode_collected = episode_collected; } public String getEpisode_updated() { return episode_updated; } public void setEpisode_updated(String episode_updated) { this.episode_updated = episode_updated; } public String getUpdate_notice() { return update_notice; } public void setUpdate_notice(String update_notice) { this.update_notice = update_notice; } public String getScore() { return score; } public void setScore(String score) { this.score = score; } public Integer getPaid() { return paid; } public void setPaid(Integer paid) { this.paid = paid; } public String getPublished() { return published; } public void setPublished(String published) { this.published = published; } public String getReleased() { return released; } public void setReleased(String released) { this.released = released; } public String getReleasedate_mainland() { return releasedate_mainland; } public void setReleasedate_mainland(String releasedate_mainland) { this.releasedate_mainland = releasedate_mainland; } public String getCreatetime() { return createtime; } public void setCreatetime(String createtime) { this.createtime = createtime; } public String getCategory() { return category; } public void setCategory(String category) { this.category = category; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public Integer getRank() { return rank; } public void setRank(Integer rank) { this.rank = rank; } public String getDouban_num() { return douban_num; } public void setDouban_num(String douban_num) { this.douban_num = douban_num; } public AttrBean getAttr() { return attr; } public void setAttr(AttrBean attr) { this.attr = attr; } public PremiumBean getPremium() { return premium; } public void setPremium(PremiumBean premium) { this.premium = premium; } public static class AttrBean { private List<DirectorBean> director; private List<PerformerBean> performer; private List<HostBean> host; public List<HostBean> getHost() { return host; } public void setHost(List<HostBean> host) { this.host = host; } public List<DirectorBean> getDirector() { return director; } public void setDirector(List<DirectorBean> director) { this.director = director; } public List<PerformerBean> getPerformer() { return performer; } public void setPerformer(List<PerformerBean> performer) { this.performer = performer; } public static class DirectorBean { /** * id : 387687 * name : 李木戈 * link : http://www.youku.com/star_page/uid_UMTU1MDc0OA==.html */ private String id; private String name; private String link; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getLink() { return link; } public void setLink(String link) { this.link = link; } } public static class HostBean { /** * id : 387687 * name : 李木戈 * link : http://www.youku.com/star_page/uid_UMTU1MDc0OA==.html */ private String id; private String name; private String link; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getLink() { return link; } public void setLink(String link) { this.link = link; } } public static class PerformerBean { /** * id : 295693 * name : 景甜 * character : 司藤 * link : http://www.youku.com/star_page/uid_UMTE4Mjc3Mg==.html */ private String id; private String name; private String character; private String link; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getCharacter() { return character; } public void setCharacter(String character) { this.character = character; } public String getLink() { return link; } public void setLink(String link) { this.link = link; } } } public static class PremiumBean { /** * onlinetime : 2021-03-07 00:00:00 * permit_duration : 2 * price : 0 */ private String onlinetime; private String permit_duration; private String price; public String getOnlinetime() { return onlinetime; } public void setOnlinetime(String onlinetime) { this.onlinetime = onlinetime; } public String getPermit_duration() { return permit_duration; } public void setPermit_duration(String permit_duration) { this.permit_duration = permit_duration; } public String getPrice() { return price; } public void setPrice(String price) { this.price = price; } } } src/main/java/com/yeshi/buwan/youku/entity/YouKuShowSimple.java
New file @@ -0,0 +1,179 @@ package com.yeshi.buwan.youku.entity; public class YouKuShowSimple { /** * id : 1e61efbfbdefbfbd04ef * name : 司藤 * link : http://www.youku.com/show_page/id_z1e61efbfbdefbfbd04ef.html * play_link : http://v.youku.com/v_show/id_XNTAzNDM2MDY5Ng==.html * last_play_link : http://v.youku.com/v_show/id_XNTExODc3MjM0MA==.html * poster : http://r1.ykimg.com/050D00006045B5A313EA3508551952B5 * thumbnail : http://r1.ykimg.com/050B00006045B5AE13EA350855E2A59A * bigthumbnail : http://r1.ykimg.com/050C00006045B5AE13EA350855E2A59A * completed : 0 * episode_count : 30 * episode_updated : 22 * category : 电视剧 * score : 0.000 * paid : 1 * published : 2021-03-08 * released : 2021-03-08 * lastupdate : 2021-03-23 20:00:06 */ private String id;//节目ID private String name;//节目名称 private String link;//节目链接 private String play_link; private String last_play_link; private String poster; private String thumbnail; private String bigthumbnail; private int completed;//是否完结 0-未完结 1-已完结 private String episode_count;//总集数 private String episode_updated;//更新至 private String category;//分类 private String score; private int paid;//是否付费 0.否 1.是 private String published;//优酷发行时间 private String released;//节目发行时间 private String lastupdate; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getLink() { return link; } public void setLink(String link) { this.link = link; } public String getPlay_link() { return play_link; } public void setPlay_link(String play_link) { this.play_link = play_link; } public String getLast_play_link() { return last_play_link; } public void setLast_play_link(String last_play_link) { this.last_play_link = last_play_link; } public String getPoster() { return poster; } public void setPoster(String poster) { this.poster = poster; } public String getThumbnail() { return thumbnail; } public void setThumbnail(String thumbnail) { this.thumbnail = thumbnail; } public String getBigthumbnail() { return bigthumbnail; } public void setBigthumbnail(String bigthumbnail) { this.bigthumbnail = bigthumbnail; } public int getCompleted() { return completed; } public void setCompleted(int completed) { this.completed = completed; } public String getEpisode_count() { return episode_count; } public void setEpisode_count(String episode_count) { this.episode_count = episode_count; } public String getEpisode_updated() { return episode_updated; } public void setEpisode_updated(String episode_updated) { this.episode_updated = episode_updated; } public String getCategory() { return category; } public void setCategory(String category) { this.category = category; } public String getScore() { return score; } public void setScore(String score) { this.score = score; } public int getPaid() { return paid; } public void setPaid(int paid) { this.paid = paid; } public String getPublished() { return published; } public void setPublished(String published) { this.published = published; } public String getReleased() { return released; } public void setReleased(String released) { this.released = released; } public String getLastupdate() { return lastupdate; } public void setLastupdate(String lastupdate) { this.lastupdate = lastupdate; } } src/main/java/com/yeshi/buwan/youku/entity/YouKuVideo.java
New file @@ -0,0 +1,179 @@ package com.yeshi.buwan.youku.entity; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.util.Date; @Document(collection = "youKuVideo") public class YouKuVideo { /** * total_up : 2 * thumbnail : https://m.ykimg.com/054201015E378F9C0F954C9BE788E356 * link : http://v.youku.com/v_show/id_XNDUzMDg2MjcyOA==.html * videoid : XNDUzMDg2MjcyOA== * total_comment : 14 * operation_limit : ["DOWNLOAD_DISABLED"] * title : 蓝盈莹凌潇肃《最爱》 节奏把控完美感人至深 * total_vv : 133909 * seconds : 6779.92 * show_videostage : 20171125 * streamtypes : ["3gphd","flvhd","hd","hd2","hd3","mp5hd4v3"] * thumbnail_v2 : https://m.ykimg.com/054101015E378F9C0F954C9BE788E356 * rc_title : null * publishtime : 2020-02-03 11:18:16 * download_status : VIP_ALLOW * state : normal * category : 综艺 * show_videoseq : 5 * is_panorama : 0 * total_vv_fmt : 13.4万 */ @Id private String videoid; private String thumbnail; private String link; @Indexed private String showId;//节目ID private String title; private String seconds; private String show_videostage; private String thumbnail_v2; private String rc_title; private String publishtime; private String download_status; private String state; private String category; private Integer is_panorama; private Date updateTime; public Date getUpdateTime() { return updateTime; } public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } public String getShowId() { return showId; } public void setShowId(String showId) { this.showId = showId; } public String getThumbnail() { return thumbnail; } public void setThumbnail(String thumbnail) { this.thumbnail = thumbnail; } public String getLink() { return link; } public void setLink(String link) { this.link = link; } public String getVideoid() { return videoid; } public void setVideoid(String videoid) { this.videoid = videoid; } public String getTitle() { return title; } public void setTitle(String title) { this.title = title; } public String getSeconds() { return seconds; } public void setSeconds(String seconds) { this.seconds = seconds; } public String getShow_videostage() { return show_videostage; } public void setShow_videostage(String show_videostage) { this.show_videostage = show_videostage; } public String getThumbnail_v2() { return thumbnail_v2; } public void setThumbnail_v2(String thumbnail_v2) { this.thumbnail_v2 = thumbnail_v2; } public String getRc_title() { return rc_title; } public void setRc_title(String rc_title) { this.rc_title = rc_title; } public String getPublishtime() { return publishtime; } public void setPublishtime(String publishtime) { this.publishtime = publishtime; } public String getDownload_status() { return download_status; } public void setDownload_status(String download_status) { this.download_status = download_status; } public String getState() { return state; } public void setState(String state) { this.state = state; } public String getCategory() { return category; } public void setCategory(String category) { this.category = category; } public Integer getIs_panorama() { return is_panorama; } public void setIs_panorama(Integer is_panorama) { this.is_panorama = is_panorama; } } src/test/java/com/hxh/spring/test/GeneralTest.java
@@ -8,6 +8,8 @@ import com.yeshi.buwan.domain.system.SystemConfig; import com.yeshi.buwan.domain.video.VideoResourceVersionMap; import com.yeshi.buwan.service.inter.video.VideoResourceVersionMapService; import com.yeshi.buwan.youku.entity.YouKuShowDetail; import com.yeshi.buwan.youku.entity.YouKuVideo; import org.yeshi.utils.generater.SpringComponentGenerater; import org.yeshi.utils.generater.entity.MongoDBDaoData; import org.yeshi.utils.generater.entity.ServiceData; @@ -17,7 +19,7 @@ @org.junit.Test public void createDao() { try { SpringComponentGenerater.createMongoDao(new MongoDBDaoData.Builder().setBaseDaoClass(MongodbBaseDao.class).setDaoPackageName("com.yeshi.buwan.dao.recommend").setEntityClass(SuperHomeRecommendSpecial.class).create(), "D:\\workspace\\BuWan\\src\\main\\java\\com\\yeshi\\buwan\\dao\\recommend\\"); SpringComponentGenerater.createMongoDao(new MongoDBDaoData.Builder().setBaseDaoClass(MongodbBaseDao.class).setDaoPackageName("com.yeshi.buwan.dao.juhe.youku").setEntityClass(YouKuVideo.class).create(), "D:\\workspace\\BuWan\\src\\main\\java\\com\\yeshi\\buwan\\dao\\juhe\\youku\\"); } catch (Exception e) { e.printStackTrace(); } src/test/java/com/hxh/spring/test/LogTest.java
@@ -110,7 +110,7 @@ public void videoPlay() throws Exception { Set<String> sets = new HashSet<>(); Gson gson = new Gson(); Scanner scanner = new Scanner(new FileInputStream("C:\\Users\\Administrator\\Desktop\\日志\\布丸播放\\play_2021_03_20.log")); Scanner scanner = new Scanner(new FileInputStream("C:\\Users\\Administrator\\Desktop\\日志\\布丸播放\\play_2021_03_23.log")); Map<String, Integer> countMap = new HashMap<>(); List<String> videoList = new ArrayList<>(); while (scanner.hasNextLine()) { src/test/java/com/hxh/spring/test/PPTVTest.java
@@ -30,9 +30,9 @@ import java.util.*; @RunWith(SpringJUnit4ClassRunner.class) //使用junit4进行测试 @ContextConfiguration(locations = {"classpath:spring.xml"}) @WebAppConfiguration //@RunWith(SpringJUnit4ClassRunner.class) //使用junit4进行测试 //@ContextConfiguration(locations = {"classpath:spring.xml"}) //@WebAppConfiguration public class PPTVTest { @Resource @@ -102,7 +102,7 @@ @Test public void getApiDetail() { PPTVApiUtil.getDetail("32159127,10215224,15326213"); PPTVApiUtil.getDetail("9040633"); } src/test/java/com/hxh/spring/test/video/YouKuTest.java
New file @@ -0,0 +1,31 @@ package com.hxh.spring.test.video; import com.yeshi.buwan.job.video.YouKuVideoUpdate; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.web.WebAppConfiguration; import javax.annotation.Resource; @RunWith(SpringJUnit4ClassRunner.class) //使用junit4进行测试 @ContextConfiguration(locations = {"classpath:spring.xml"}) @WebAppConfiguration public class YouKuTest { @Resource private YouKuVideoUpdate youKuVideoUpdate; @Test public void test1() { try { youKuVideoUpdate.updateLatestVideo("动漫"); } catch (Exception e) { e.printStackTrace(); } } }