package com.newvideo.youku;
|
|
import java.io.IOException;
|
import java.io.UnsupportedEncodingException;
|
import java.util.ArrayList;
|
import java.util.Calendar;
|
import java.util.Collections;
|
import java.util.Comparator;
|
import java.util.List;
|
import java.util.Map;
|
|
import javax.annotation.Resource;
|
|
import org.hibernate.HibernateException;
|
import org.hibernate.Session;
|
import org.jsoup.Jsoup;
|
import org.jsoup.nodes.Document;
|
import org.jsoup.select.Elements;
|
import org.springframework.orm.hibernate4.HibernateCallback;
|
import org.springframework.stereotype.Component;
|
|
import com.newvideo.dao.CategoryVideoDao;
|
import com.newvideo.domain.CategoryVideo;
|
import com.newvideo.domain.VideoDetailInfo;
|
import com.newvideo.domain.VideoInfo;
|
import com.newvideo.service.imp.VideoResourceService;
|
import com.newvideo.service.imp.VideoService;
|
import com.newvideo.service.imp.juhe.YouKuService;
|
import com.newvideo.util.JuheVideoUtil;
|
import com.newvideo.util.NumberUtil;
|
import com.newvideo.util.StringUtil;
|
import com.newvideo.util.TimeUtil;
|
import com.newvideo.youku.entity.Program;
|
import com.newvideo.youku.entity.ProgramVideo;
|
|
import net.sf.json.JSONArray;
|
import net.sf.json.JSONObject;
|
|
@Component
|
public class YouKuUtil {
|
@Resource
|
private YouKuService youKuService;
|
@Resource
|
private VideoService videoService;
|
@Resource
|
private VideoResourceService videoResourceService;
|
|
@Resource
|
private CategoryVideoDao categoryVideoDao;
|
|
public VideoResourceService getVideoResourceService() {
|
return videoResourceService;
|
}
|
|
public void setVideoResourceService(VideoResourceService videoResourceService) {
|
this.videoResourceService = videoResourceService;
|
}
|
|
public VideoService getVideoService() {
|
return videoService;
|
}
|
|
public void setVideoService(VideoService videoService) {
|
this.videoService = videoService;
|
}
|
|
public YouKuService getYouKuService() {
|
return youKuService;
|
}
|
|
public void setYouKuService(YouKuService youKuService) {
|
this.youKuService = youKuService;
|
}
|
|
public static VideoDetailInfo convertProgramVideoToVideoDetail(ProgramVideo pv, Program p) {
|
VideoDetailInfo vinfo = new VideoDetailInfo();
|
vinfo.setExtraId(pv.getId());
|
vinfo.setName(pv.getRc_title());
|
if ("纪录片".equalsIgnoreCase(p.getCategory()) || "综艺".equalsIgnoreCase(p.getCategory()))
|
vinfo.setTag(pv.getTitle());
|
else
|
vinfo.setTag(pv.getStage());
|
vinfo.setType("youku");
|
return vinfo;
|
}
|
|
public static List<ProgramVideo> getProgramVideoList(String pid) {
|
return programVideoList(pid);
|
}
|
|
public static VideoInfo convertProgramToVideoInfo(Program p) {
|
String time = p.getReleased();
|
Calendar ca = Calendar.getInstance();
|
ca.setTimeInMillis(TimeUtil.convertDateToTemp(time));
|
int year = ca.get(Calendar.YEAR);
|
int month = ca.get(Calendar.MONTH) + 1;
|
int day = ca.get(Calendar.DAY_OF_MONTH);
|
String director = "";
|
String mainActor = "";
|
if (!StringUtil.isNullOrEmpty(p.getAttrStr())) {
|
JSONObject root = JSONObject.fromObject(p.getAttrStr());
|
JSONArray da = root.optJSONArray("director");
|
if (da != null)
|
for (int i = 0; i < da.size(); i++) {
|
director += da.optJSONObject(i).optString("name") + " ";
|
}
|
|
if (director.endsWith(" "))
|
director = director.substring(0, director.length() - 1);
|
|
da = root.optJSONArray("performer");
|
if (da != null)
|
for (int i = 0; i < da.size(); i++) {
|
mainActor += da.optJSONObject(i).optString("name") + " ";
|
}
|
|
mainActor = (mainActor + "").trim();
|
}
|
String score = NumberUtil.get1PointNumber(Double.parseDouble(p.getScore()));
|
|
String tag = "";
|
if (p.getCategory().contains("电影")) {
|
tag = "评分：" + score;
|
} else if (p.getCategory().contains("电视剧")) {
|
if (p.getCompleted() == 0) {// 未更新完
|
tag = "更新至" + p.getEpisode_updated() + "集";
|
} else if (p.getCompleted() == 1) {// 更新完
|
tag = p.getEpisode_count() + "集全";
|
}
|
} else if (p.getCategory().contains("综艺")) {
|
Comparator<ProgramVideo> cm = new Comparator<ProgramVideo>() {
|
public int compare(ProgramVideo o1, ProgramVideo o2) {
|
long t2 = Long.parseLong(o2.getStage());
|
long t1 = Long.parseLong(o1.getStage());
|
return (int) (t2 - t1);// 倒序排列
|
}
|
};
|
Collections.sort(p.getVideoList(), cm);
|
String t = "";
|
long ti = TimeUtil.convertGernalTime(p.getVideoList().get(0).getStage(), "yyyyMMdd");
|
if (ti > 0)
|
t = TimeUtil.getGernalTime(ti, "yyyy-MM-dd");
|
else
|
t = p.getVideoList().get(0).getStage();
|
tag = "更新至" + t;
|
} else if (p.getCategory().contains("动漫")) {
|
if (Integer.parseInt(p.getEpisode_count()) <= 1) {
|
tag = "评分：" + score;
|
} else {
|
if (p.getCompleted() == 0) {// 未更新完
|
tag = "更新至" + p.getEpisode_updated() + "集";
|
} else if (p.getCompleted() == 1) {// 更新完
|
tag = p.getEpisode_count() + "集全";
|
}
|
}
|
} else if (p.getCategory().contains("纪录片") || p.getCategory().contains("纪实")) {
|
Comparator<ProgramVideo> cm = new Comparator<ProgramVideo>() {
|
public int compare(ProgramVideo o1, ProgramVideo o2) {
|
long t2 = Long.parseLong(o2.getStage());
|
long t1 = Long.parseLong(o1.getStage());
|
return (int) (t2 - t1);// 倒序排列
|
}
|
};
|
Collections.sort(p.getVideoList(), cm);
|
String t = "";
|
long ti = TimeUtil.convertGernalTime(p.getVideoList().get(0).getStage(), "yyyyMMdd");
|
if (ti > 0)
|
t = TimeUtil.getGernalTime(ti, "yyyy-MM-dd");
|
else
|
t = p.getVideoList().get(0).getStage();
|
tag = "更新至" + t;
|
|
}
|
|
Comparator<ProgramVideo> cm = new Comparator<ProgramVideo>() {
|
public int compare(ProgramVideo o1, ProgramVideo o2) {
|
long t2 = TimeUtil.convertGernalTime(o2.getPublished(), "yyyy-MM-dd HH:mm:ss");
|
long t1 = TimeUtil.convertGernalTime(o1.getPublished(), "yyyy-MM-dd HH:mm:ss");
|
return (int) (t2 - t1);// 倒序排列
|
}
|
};
|
|
Collections.sort(p.getVideoList(), cm);
|
String latestHpicture = StringUtil.isNullOrEmpty(p.getVideoList().get(0).getBigThumbnail())
|
? p.getVideoList().get(0).getThumbnail_v2() : p.getVideoList().get(0).getBigThumbnail();
|
long updateTime = TimeUtil.convertGernalTime(time, "yyyy-MM-dd");
|
|
VideoInfo vi = new VideoInfo();
|
vi.setArea(p.getArea());
|
vi.setBaseurl(p.getLink());
|
vi.setCanSave(false);
|
vi.setCommentCount(0);
|
vi.setContentType(1);
|
vi.setCreatetime(System.currentTimeMillis());
|
vi.setDay(day + "");
|
vi.setDirector(director);
|
vi.setFocus("");
|
vi.setHpicture(p.getThumbnail_large());
|
vi.setIntroduction(p.getDescription());
|
vi.setLatestHpicture(latestHpicture);
|
vi.setLatestVpicture("");
|
vi.setMainActor(mainActor);
|
vi.setMonth(month + "");
|
vi.setName(p.getName());
|
vi.setNowNumber(0 + "");
|
vi.setOrderby("0");
|
vi.setPicture(p.getPoster_large());
|
vi.setScore(score);
|
vi.setShare("0");
|
vi.setShow(1 + "");
|
vi.setTag(tag);
|
vi.setVpicture(p.getPoster_large());
|
vi.setWatchCount(0 + "");
|
vi.setYear(year + "");
|
vi.setUpdatetime(updateTime + "");
|
vi.setVideocount(p.getVideoList() != null ? p.getVideoList().size() : 1);
|
return vi;
|
}
|
|
public static VideoInfo convertVideoToVideoInfo(ProgramVideo p) {
|
String time = p.getPublished();
|
Calendar ca = Calendar.getInstance();
|
ca.setTimeInMillis(TimeUtil.convertDateToTemp2(time));
|
int year = ca.get(Calendar.YEAR);
|
int month = ca.get(Calendar.MONTH) + 1;
|
int day = ca.get(Calendar.DAY_OF_MONTH);
|
String score = "9.0";
|
String tag = "";
|
if (!StringUtil.isNullOrEmpty(p.getDuration())) {
|
tag = NumberUtil.convertSecondToString((int) Double.parseDouble(p.getDuration()));
|
}
|
long updateTime = ca.getTimeInMillis();
|
VideoInfo vi = new VideoInfo();
|
vi.setArea("");
|
vi.setBaseurl(p.getLink());
|
vi.setCanSave(false);
|
vi.setCommentCount(0);
|
vi.setContentType(1);
|
vi.setCreatetime(System.currentTimeMillis());
|
vi.setDay(day + "");
|
vi.setDirector("");
|
vi.setFocus("");
|
vi.setHpicture(StringUtil.isNullOrEmpty(p.getBigThumbnail()) ? p.getThumbnail() : p.getBigThumbnail());
|
vi.setIntroduction("");
|
vi.setLatestHpicture("");
|
vi.setLatestVpicture("");
|
vi.setMainActor("");
|
vi.setMonth(month + "");
|
vi.setName(p.getTitle());
|
vi.setNowNumber(0 + "");
|
vi.setOrderby("0");
|
vi.setPicture(p.getThumbnail());
|
vi.setScore(score);
|
vi.setShare("0");
|
vi.setShow(1 + "");
|
vi.setTag(tag);
|
vi.setVpicture(StringUtil.isNullOrEmpty(p.getBigThumbnail()) ? p.getThumbnail() : p.getBigThumbnail());
|
vi.setWatchCount(0 + "");
|
vi.setYear(year + "");
|
vi.setUpdatetime(updateTime + "");
|
vi.setVideocount(1);
|
return vi;
|
}
|
|
public static int getProgramType(Program p, String genra) {
|
if (p.getCategory().equalsIgnoreCase("电影")) {
|
if ("短片".equalsIgnoreCase(genra)) {
|
return 281;
|
} else if ("优酷出品".equalsIgnoreCase(genra)) {
|
return 281;
|
} else if ("纪录片".equalsIgnoreCase(genra))// 科技->纪录片
|
return 284;
|
return 151;
|
|
} else if (p.getCategory().equalsIgnoreCase("电视剧")) {
|
return 150;
|
} else if (p.getCategory().equalsIgnoreCase("综艺")) {
|
// 排除 美食 旅游 汽车 时尚 搞笑
|
if (!genra.equalsIgnoreCase("美食") && !genra.equalsIgnoreCase("旅游") && !genra.equalsIgnoreCase("汽车")
|
&& !genra.equalsIgnoreCase("时尚") && !genra.equalsIgnoreCase("搞笑")
|
&& !genra.equalsIgnoreCase("优酷出品")) {
|
JSONObject tv = JSONObject.fromObject(p.getAttrStr()).optJSONObject("tv_station");
|
// 是否包含电视台
|
if (tv != null && !StringUtil.isNullOrEmpty(tv.optString("name"))) {// 直接加入综艺
|
return 152;
|
} else {
|
if (genra.equalsIgnoreCase("生活"))
|
return 230;
|
|
if (genra.equalsIgnoreCase("脱口秀"))
|
return 285;
|
|
// 不是电视台的,待定
|
return 0;
|
}
|
} else {// 处理 美食 旅游 汽车 时尚 搞笑
|
if (genra.equalsIgnoreCase("搞笑"))
|
return 216;
|
if (genra.equalsIgnoreCase("美食")) {
|
JSONObject tv = JSONObject.fromObject(p.getAttrStr()).optJSONObject("tv_station");
|
// 不包含电视台
|
if (tv == null || StringUtil.isNullOrEmpty(tv.optString("name"))) {
|
return 228;
|
}
|
}
|
|
if (genra.equalsIgnoreCase("旅游")) {
|
JSONObject tv = JSONObject.fromObject(p.getAttrStr()).optJSONObject("tv_station");
|
// 不包含电视台
|
if (tv == null || StringUtil.isNullOrEmpty(tv.optString("name"))) {
|
return 229;
|
}
|
}
|
|
if (genra.equalsIgnoreCase("汽车")) {
|
JSONObject tv = JSONObject.fromObject(p.getAttrStr()).optJSONObject("tv_station");
|
// 不包含电视台
|
if (tv == null || StringUtil.isNullOrEmpty(tv.optString("name"))) {
|
return 226;
|
}
|
}
|
|
if (genra.equalsIgnoreCase("时尚")) {
|
JSONObject tv = JSONObject.fromObject(p.getAttrStr()).optJSONObject("tv_station");
|
// 不包含电视台
|
if (tv == null || StringUtil.isNullOrEmpty(tv.optString("name"))) {
|
return 232;
|
}
|
}
|
|
if (genra.equalsIgnoreCase("优酷出品")) {
|
return 219;
|
}
|
|
}
|
|
} else if (p.getCategory().equalsIgnoreCase("动漫")) {
|
return 153;
|
} else if (p.getCategory().equalsIgnoreCase("音乐")) {// 此处不处理, 音乐放在小视频中处理
|
|
} else if (p.getCategory().equalsIgnoreCase("纪录片")) {
|
if (!genra.equalsIgnoreCase("军事"))
|
return 284;
|
else
|
return 223;
|
|
} else if (p.getCategory().equalsIgnoreCase("教育")) {
|
return 225;
|
}
|
return 0;
|
}
|
|
public static List<Integer> getProgramVideoType(ProgramVideo p, String genra) {
|
List<Integer> list = new ArrayList<Integer>();
|
|
if (p.getCategory().equalsIgnoreCase("娱乐")) {
|
if (genra.equalsIgnoreCase("曲艺"))
|
list.add(217);
|
else if (genra.equalsIgnoreCase("戏剧"))
|
// list.add(0);
|
;
|
else if (genra.equalsIgnoreCase("曲艺"))
|
;
|
else if (genra.equalsIgnoreCase("艺术"))
|
;
|
else if (genra.equalsIgnoreCase("魔术"))
|
list.add(286);
|
else
|
list.add(219);
|
|
} else if (p.getCategory().equalsIgnoreCase("资讯")) {
|
if (genra.equalsIgnoreCase("科技资讯")) {
|
list.add(222);
|
list.add(244);
|
} else if (genra.equalsIgnoreCase("军事资讯")) {
|
list.add(223);
|
list.add(243);
|
} else if (genra.equalsIgnoreCase("财经资讯")) {
|
list.add(289);
|
list.add(227);
|
} else if (genra.equalsIgnoreCase("社会资讯")) {
|
list.add(242);
|
} else if (genra.equalsIgnoreCase("生活资讯"))
|
list.add(288);
|
else if (genra.equalsIgnoreCase("时政资讯"))
|
list.add(290);
|
else if (genra.equalsIgnoreCase("法制"))
|
list.add(291);
|
else
|
list.add(159);
|
|
} else if (p.getCategory().equalsIgnoreCase("体育")) {
|
if (genra.equalsIgnoreCase("健身")) {
|
list.add(222);
|
list.add(235);
|
} else if (genra.equalsIgnoreCase("体育资讯")) {
|
list.add(287);
|
} else if (genra.equalsIgnoreCase("极限运动")) {
|
list.add(234);
|
} else if (genra.equalsIgnoreCase("跑酷")) {
|
list.add(234);
|
} else if (genra.equalsIgnoreCase("自行车")) {
|
list.add(234);
|
} else if (genra.equalsIgnoreCase("轮滑")) {
|
list.add(234);
|
} else if (genra.equalsIgnoreCase("滑板")) {
|
list.add(234);
|
} else if (genra.equalsIgnoreCase("篮球")) {
|
list.add(292);
|
} else if (genra.equalsIgnoreCase("CBA")) {
|
list.add(292);
|
} else if (genra.equalsIgnoreCase("足球")) {
|
list.add(292);
|
} else if (genra.equalsIgnoreCase("英超")) {
|
list.add(292);
|
} else if (genra.equalsIgnoreCase("大球运动")) {
|
list.add(292);
|
} else if (genra.equalsIgnoreCase("网球")) {
|
list.add(292);
|
} else if (genra.equalsIgnoreCase("小球运动")) {
|
list.add(293);
|
} else if (genra.equalsIgnoreCase("赛车")) {
|
list.add(294);
|
} else if (genra.equalsIgnoreCase("模型")) {
|
list.add(294);
|
} else if (genra.equalsIgnoreCase("武术")) {
|
list.add(295);
|
} else if (genra.equalsIgnoreCase("格斗")) {
|
list.add(295);
|
} else if (genra.equalsIgnoreCase("田径")) {
|
list.add(296);
|
} else if (genra.equalsIgnoreCase("水上项目")) {
|
list.add(296);
|
} else if (genra.equalsIgnoreCase("力量项目")) {
|
list.add(296);
|
} else if (genra.equalsIgnoreCase("冰雪项目")) {
|
list.add(296);
|
} else if (genra.equalsIgnoreCase("棋牌")) {
|
list.add(297);
|
} else if (genra.equalsIgnoreCase("射击")) {
|
list.add(273);
|
} else if (genra.equalsIgnoreCase("体育舞蹈")) {
|
list.add(273);
|
} else if (genra.equalsIgnoreCase("技巧")) {
|
list.add(273);
|
} else if (genra.equalsIgnoreCase("其他")) {
|
list.add(273);
|
}
|
|
} else if (p.getCategory().equalsIgnoreCase("游戏")) {
|
if (genra.equalsIgnoreCase("游戏资讯")) {
|
list.add(272);
|
} else if (genra.equalsIgnoreCase("游戏达人")) {
|
list.add(212);
|
} else if (genra.equalsIgnoreCase("魔兽世界")) {
|
list.add(212);
|
} else if (genra.equalsIgnoreCase("地下城与勇士")) {
|
list.add(212);
|
} else if (genra.equalsIgnoreCase("单机游戏")) {
|
list.add(213);
|
} else if (genra.equalsIgnoreCase("手机游戏")) {
|
list.add(214);
|
} else if (genra.equalsIgnoreCase("游戏音乐")) {
|
list.add(298);
|
}
|
//
|
|
} else if (p.getCategory().equalsIgnoreCase("搞笑")) {
|
if (genra.equalsIgnoreCase("搞笑动物")) {
|
list.add(216);
|
list.add(220);
|
} else
|
list.add(216);
|
|
} else if (p.getCategory().equalsIgnoreCase("生活")) {
|
if (genra.equalsIgnoreCase("美食")) {
|
list.add(228);
|
} else if (genra.equalsIgnoreCase("居家")) {
|
list.add(230);
|
} else if (genra.equalsIgnoreCase("健康")) {
|
list.add(230);
|
} else if (genra.equalsIgnoreCase("家居")) {
|
list.add(230);
|
}
|
} else if (p.getCategory().equalsIgnoreCase("汽车")) {
|
list.add(226);
|
} else if (p.getCategory().equalsIgnoreCase("科技")) {
|
list.add(221);
|
} else if (p.getCategory().equalsIgnoreCase("时尚")) {
|
list.add(232);
|
} else if (p.getCategory().equalsIgnoreCase("旅游")) {
|
list.add(229);
|
} else if (p.getCategory().equalsIgnoreCase("音乐")) {
|
if (genra.equalsIgnoreCase("音乐MV")) {
|
list.add(301);
|
} else if (genra.equalsIgnoreCase("现场版")) {
|
list.add(301);
|
} else if (genra.equalsIgnoreCase("演唱会")) {
|
list.add(301);
|
} else if (genra.equalsIgnoreCase("电影原声")) {
|
list.add(279);
|
} else if (genra.equalsIgnoreCase("电视剧原声")) {
|
list.add(279);
|
} else if (genra.equalsIgnoreCase("动漫音乐")) {
|
list.add(300);
|
} else if (genra.equalsIgnoreCase("游戏音乐")) {
|
list.add(300);
|
} else if (genra.equalsIgnoreCase("广告音乐")) {
|
list.add(280);
|
}
|
|
}
|
|
if (list.size() == 0)// 其他无用的加入杂类
|
list.add(299);
|
|
return list;
|
}
|
|
// 节目列表
|
public static Map<String, Object> programList(String cate, String genre, int year, int page) {
|
try {
|
Map<String, Object> map = YouKuApi.getProgramList(cate, genre, year, page);
|
System.out.println(cate + "--" + genre + "--" + year + "--" + map.get("total"));
|
// System.out.println(cate + "--" + genre + "--"
|
// + ((List<Program>) map.get("data")).size());
|
return map;
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
return null;
|
}
|
|
@SuppressWarnings("unchecked")
|
public static List<ProgramVideo> programVideoList(String id) {
|
List<ProgramVideo> list = new ArrayList<ProgramVideo>();
|
int page = 1;
|
Map<String, Object> map = YouKuApi.getProgramVideo(id, page);
|
List<ProgramVideo> li = (List<ProgramVideo>) map.get("data");
|
if (li != null && li.size() > 0)
|
list.addAll(li);
|
while (li != null && li.size() > 0) {
|
page++;
|
map = YouKuApi.getProgramVideo(id, page);
|
li = (List<ProgramVideo>) map.get("data");
|
list.addAll(li);
|
}
|
int freeCount = YouKuApi.getProgramNotPayCount(id);
|
if (list.size() > freeCount) {
|
list = list.subList(0, freeCount);
|
}
|
return list;
|
}
|
|
// 解析节目信息
|
public void startParseShow() {
|
final int[] years = new int[] { 2016, 2015, 2014, 2013, 2012, 2011, 2010, 2009, 2008, 2007, 2006, 2005, 2004,
|
2003, 2002, 2001, 2000, 1999, 1998, 1997, 1996, 1995, 1994, 1993, 1992, 1991, 1990, 1989, 1988, 1987,
|
1986, 1985, 1984, 1983, 1982, 1981, 1980, 1979, 1978, 1977, 1976, 1975, 1974, 1973, 1972, 1971, 1970 };
|
|
List<YouKuCategory> list = YouKuApi.getProgramCategory();
|
for (int y = 0; y < list.size(); y++) {
|
final YouKuCategory yc = list.get(y);
|
// if (!yc.getLabel().contains("综艺"))
|
// continue;
|
for (int c = 0; c < yc.getGenre().size(); c++) {
|
final YouKuCategory child = yc.getGenre().get(c);
|
// if (!child.getLabel().contains("搞笑"))
|
// continue;
|
|
for (int year = 0; year < 1; year++)
|
try {
|
int page = 1;
|
Map<String, Object> map = programList(yc.getLabel(), child.getLabel(), years[year], page);
|
List<Program> li = (List<Program>) map.get("data");
|
int total = Integer.parseInt(map.get("total") + "");
|
int count = 0;
|
int zeroCount = 0;
|
for (int p = 1; p < 61; p++) {
|
map = programList(yc.getLabel(), child.getLabel(), years[year], p);
|
li = (List<Program>) map.get("data");
|
count += (li != null ? li.size() : 0);
|
if (li == null || li.size() == 0)
|
zeroCount++;
|
else
|
zeroCount = 0;
|
if (zeroCount > 5)
|
break;
|
System.out.println("数量:" + count);
|
if (li != null)
|
for (int i = 0; i < li.size(); i++) {
|
li.set(i, YouKuApi.getProgramDetail(li.get(i).getId()));
|
if (StringUtil.isNullOrEmpty(li.get(i).getGenre()))
|
li.get(i).setGenre(child.getLabel());
|
li.get(i).setVideoList(programVideoList(li.get(i).getId()));
|
|
}
|
// 加入数据库
|
for (Program pm : li) {
|
// if (!pm.getName().equalsIgnoreCase("每日一囧
|
// 2016"))
|
// continue;
|
saveProgram(pm);
|
}
|
}
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
}
|
|
// 解析小视频信息
|
public void startParseVideo() {
|
List<YouKuCategory> list = YouKuApi.getVideoCategory();
|
for (YouKuCategory ycy : list) {
|
final YouKuCategory yc = ycy;
|
if (!yc.getLabel().contains("游戏"))
|
continue;
|
// 排除7大版权分类
|
if (yc.getLabel().contains("电影") || yc.getLabel().contains("电视剧") || yc.getLabel().contains("综艺")
|
|| yc.getLabel().contains("动漫") || yc.getLabel().contains("纪实") || yc.getLabel().contains("教育"))
|
continue;
|
if (!yc.getLabel().contains("音乐")) {
|
new Thread(new Runnable() {
|
|
@SuppressWarnings("unchecked")
|
public void run() {
|
|
if (!yc.getLabel().contains("资讯"))
|
for (YouKuCategory genre : yc.getGenre()) {
|
|
try {
|
Map<String, Object> map = YouKuApi.getVideoListByCategory(yc.getLabel(),
|
genre.getLabel(), 1);
|
int total = Integer.parseInt(map.get("total") + "");
|
if (total > 2000)
|
total = 2000;
|
|
if (total < 400)
|
total = 400;
|
|
int page = total % 50 == 0 ? total / 50 : total / 50 + 1;
|
for (int n = 0; n < 3; n++) {// 做三次重复爬去,防止丢失部分视频
|
for (int p = 1; p < page; p++) {
|
map = YouKuApi.getVideoListByCategory(yc.getLabel(), genre.getLabel(), p);
|
List<ProgramVideo> videoList = (List<ProgramVideo>) map.get("data");
|
addProgramVideo(videoList, genre.getLabel());
|
}
|
}
|
|
} catch (UnsupportedEncodingException e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
}).start();
|
|
} else {
|
new Thread(new Runnable() {
|
public void run() {
|
yc.setGenre(new ArrayList<YouKuCategory>());
|
YouKuCategory child = new YouKuCategory();
|
child.setLabel("音乐MV");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("现场版");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("演唱会");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("电影原声");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("电视剧原声");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("动漫音乐");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("游戏音乐");
|
yc.getGenre().add(child);
|
child = new YouKuCategory();
|
child.setLabel("广告音乐");
|
yc.getGenre().add(child);
|
|
for (YouKuCategory genre : yc.getGenre()) {
|
for (int p = 1; p < 4; p++) {
|
List<ProgramVideo> videoList = YouKuApi.getMusicVideoList(genre.getLabel(), p);
|
addProgramVideo(videoList, genre.getLabel());
|
}
|
}
|
}
|
}).start();
|
|
}
|
}
|
|
}
|
|
// 解析小视频信息
|
public void startParseZiXun() {
|
List<YouKuCategory> list = YouKuApi.getVideoCategory();
|
for (YouKuCategory ycy : list) {
|
if (!ycy.getLabel().contains("资讯"))
|
continue;
|
|
// 获取资讯
|
for (int p = 1; p < 10; p++) {
|
Map<String, Object> map = null;
|
try {
|
map = YouKuApi.getZiXunVideoList("资讯", p);
|
} catch (UnsupportedEncodingException e) {
|
e.printStackTrace();
|
}
|
List<ProgramVideo> videoList = (List<ProgramVideo>) map.get("data");
|
System.out.println(videoList.size());
|
addProgramVideo(videoList, "资讯");
|
}
|
}
|
|
}
|
|
private void addProgramVideo(List<ProgramVideo> videoList, String genre) {
|
for (ProgramVideo pv : videoList) {
|
final String name = pv.getTitle();
|
if (!(Boolean) categoryVideoDao.excute(new HibernateCallback<Boolean>() {
|
public Boolean doInHibernate(Session session) throws HibernateException {
|
return JuheVideoUtil.isNeedAdd(session, name);
|
}
|
}))
|
continue;
|
|
System.out.println(genre);
|
// 加入videoInfo
|
try {
|
ProgramVideo video = YouKuApi.getVideoDetail(pv.getId());
|
if (video != null) {
|
pv.setCopyright_type(video.getCopyright_type());
|
pv.setPublic_type(video.getPublic_type());
|
pv.setOperationLimit(video.getOperationLimit());
|
}
|
System.out.println(pv.getCopyright_type() + "-" + pv.getPublic_type());
|
if ("all".equalsIgnoreCase(pv.getPublic_type())) {// 不是版权并且是公开的
|
youKuService.saveVideo(pv);// 存入数据库
|
youKuService.addVideoToVideoInfo(pv, genre);
|
}
|
} catch (Exception e) {
|
|
}
|
}
|
}
|
|
public void saveProgram(Program pm) {
|
final String name = pm.getName();
|
if (!(Boolean) categoryVideoDao.excute(new HibernateCallback<Boolean>() {
|
public Boolean doInHibernate(Session session) throws HibernateException {
|
return JuheVideoUtil.isNeedAdd(session, name);
|
}
|
}))
|
return;
|
|
ProgramVideo pv = youKuService.getLatestProgramVideo(pm.getId());
|
Comparator<ProgramVideo> cp = new Comparator<ProgramVideo>() {
|
public int compare(ProgramVideo o1, ProgramVideo o2) {
|
long t1 = Long.parseLong(o1.getStage());
|
long t2 = Long.parseLong(o2.getStage());
|
return (int) (t1 - t2);
|
}
|
};
|
|
Collections.sort(pm.getVideoList(), cp);
|
List<ProgramVideo> vlist = new ArrayList<ProgramVideo>();
|
|
boolean isUpdate = false;
|
if (pm.getVideoList().size() > 0 && pv != null)
|
isUpdate = Long.parseLong(pm.getVideoList().get(pm.getVideoList().size() - 1).getStage()) > Long
|
.parseLong(pv.getStage());
|
if (pv == null || isUpdate) {
|
if (pv == null)// 首次加入
|
youKuService.saveProgram(pm, pm.getVideoList());// 保存数据到本地
|
else {
|
for (int i = 0; i < pm.getVideoList().size(); i++) {
|
if (pm.getVideoList().get(i).getId().equalsIgnoreCase(pv.getId())) {
|
vlist = pm.getVideoList().subList(i, pm.getVideoList().size());
|
break;
|
}
|
}
|
youKuService.saveProgram(pm, vlist);
|
}
|
}
|
|
if (pm.getVideoList() != null && pm.getVideoList().size() > 0)
|
if (!"original".equalsIgnoreCase(pm.getVideoList().get(0).getCopyright_type()))
|
youKuService.addProgramToVideoInfo(pm, isUpdate);
|
}
|
|
public void updateProgramById(String id) {
|
try {
|
Program pm = YouKuApi.getProgramDetail(id);
|
pm.setVideoList(programVideoList(id));
|
saveProgram(pm);
|
} catch (UnsupportedEncodingException e) {
|
e.printStackTrace();
|
}
|
|
}
|
|
public static void parseSingleProgram(String id) {
|
try {
|
Program pm = YouKuApi.getProgramDetail(id);
|
pm.setVideoList(programVideoList(pm.getId()));
|
} catch (UnsupportedEncodingException e) {
|
e.printStackTrace();
|
}
|
}
|
|
@SuppressWarnings("unchecked")
|
public static boolean isSameVideo(List<Integer> typeList, Program p, VideoInfo v, Session session) {
|
try {
|
if (p.getName().equalsIgnoreCase(v.getName())) {
|
// 判断是否含有优酷来源
|
Object c = session
|
.createQuery("select count(*) from ResourceVideo rv where rv.video.id=? and rv.resource.id=15")
|
.setParameter(0, v.getId()).uniqueResult();
|
if (Integer.parseInt(c + "") > 0) {
|
VideoInfo tempVideo = convertProgramToVideoInfo(p);
|
if (v.getPicture().equalsIgnoreCase(tempVideo.getPicture())
|
&& (tempVideo.getYear() + "").equalsIgnoreCase(v.getYear()))// 封面图标相同且是同一年份则判断相同
|
return true;
|
}
|
|
if (p.getCategory().contains("电影") || p.getCategory().contains("电视剧") || p.getCategory().contains("综艺")
|
|| p.getCategory().contains("动漫") || p.getCategory().contains("纪录片")) {
|
String time = p.getReleased();
|
Calendar ca = Calendar.getInstance();
|
ca.setTimeInMillis(TimeUtil.convertDateToTemp(time));
|
int year = ca.get(Calendar.YEAR);
|
String wheres = "";
|
for (Integer type : typeList)
|
wheres += String.format(" cv.videoType.id=%s or", type + "");
|
if (wheres.endsWith("or"))
|
wheres = wheres.substring(0, wheres.length() - 2);
|
List<CategoryVideo> cvList = session
|
.createQuery("from CategoryVideo cv where cv.video.id=? and (" + wheres + ")")
|
.setParameter(0, v.getId()).list();
|
boolean isSame = false;
|
if (cvList == null || cvList.size() == 0)
|
isSame = false;
|
else
|
isSame = true;
|
|
if (Math.abs(
|
year - Integer.parseInt(StringUtil.isNullOrEmpty(v.getYear()) ? "2016" : v.getYear())) < 3
|
&& isSame)
|
return true;
|
} else {
|
return true;
|
}
|
}
|
return false;
|
} catch (Exception e) {
|
return false;
|
}
|
}
|
|
public static String parseShowIdByUrl(String url) {
|
try {
|
Document doc = Jsoup.connect(url).timeout(60 * 1000)
|
.userAgent(
|
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.59 QQ/7.8.16379.201 Safari/537.36")
|
.get();
|
Elements els = doc.getElementsByTag("script");
|
for (int i = 0; i < els.size(); i++) {
|
if (els.get(i).toString().contains("var showid_en")) {
|
String id = els.get(i).toString().split("var showid_en=")[1].split(";")[0].replace("\"", "");
|
return id;
|
}
|
}
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
return "";
|
}
|
|
public static List<String> getProgramIdList(String type, int page) {
|
List<String> idList = new ArrayList<String>();
|
String url = "";
|
if (type.equalsIgnoreCase("电影")) {
|
url = String.format("http://list.youku.com/category/show/c_96_pt_1_s_6_d_1_p_%s.html?spm=0.0.0.0.djf5YC",
|
page + "");
|
} else if (type.equalsIgnoreCase("电视剧")) {
|
url = String.format("http://list.youku.com/category/show/c_97_pt_1_s_6_d_1_p_%s.html?spm=0.0.0.0.djf5YC",
|
page + "");
|
} else if (type.equalsIgnoreCase("综艺")) {
|
url = String.format("http://list.youku.com/category/show/c_85_pt_1_s_6_d_1_p_%s.html?spm=0.0.0.0.djf5YC",
|
page + "");
|
} else if (type.equalsIgnoreCase("动漫")) {
|
url = String.format("http://list.youku.com/category/show/c_100_pt_1_s_6_d_1_p_%s.html?spm=0.0.0.0.djf5YC",
|
page + "");
|
}
|
|
Document doc = null;
|
try {
|
doc = Jsoup.connect(url).timeout(10 * 1000)
|
.userAgent(
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25")
|
.get();
|
} catch (IOException e1) {
|
// TODO Auto-generated catch block
|
e1.printStackTrace();
|
}
|
|
Elements lis = doc.getElementsByAttributeValue("class", "box-series").get(0).getElementsByTag("ul").get(0)
|
.getElementsByTag("li");
|
for (int i = 0; i < lis.size(); i++) {
|
String u = lis.get(i).getElementsByTag("a").attr("href");
|
if (!StringUtil.isNullOrEmpty(u)) {
|
try {
|
Thread.sleep(10 * 1000);
|
} catch (InterruptedException e) {
|
e.printStackTrace();
|
}
|
|
String id = parseShowIdByUrl(u);
|
System.out.println(id);
|
if (!StringUtil.isNullOrEmpty(id)) {
|
boolean isC = true;
|
for (int j = 0; j < idList.size(); j++) {
|
if (idList.get(j).equalsIgnoreCase(id))
|
isC = false;
|
|
}
|
if (isC)
|
idList.add(id);
|
}
|
}
|
}
|
|
return idList;
|
}
|
|
}
|
