风行网页版爬虫

admin

2024-09-05 ab35ac8b769b2d9816dffb33a64f2c6f7bd5dd6e

 src/main/java/com/yeshi/buwan/videos/youku/YouKuApiUtil.java

@@ -5,12 +5,14 @@
import com.yeshi.buwan.videos.youku.entity.YouKuShowDetail;
import com.yeshi.buwan.videos.youku.entity.YouKuShowSimple;
import com.yeshi.buwan.videos.youku.entity.YouKuVideo;
import org.json.JSONArray;
import org.json.JSONObject;
import org.yeshi.utils.HttpUtil;
import org.yeshi.utils.StringUtil;

import java.io.UnsupportedEncodingException;
import java.lang.reflect.Type;
import java.net.URI;
import java.net.URLEncoder;
import java.util.*;

@@ -95,6 +97,55 @@
        return null;
    }

    /**
     * @return com.yeshi.buwan.videos.youku.YouKuApiUtil.ListResultDTO
     * @author hxh
     * @description 获取热门的专辑
     * @date 15:26 2024/3/21
     * @param: category
     * @param: page
     * @param: pageSize
     **/
    public static ListResultDTO getHotShowListByCategory(String category, int page, int pageSize) throws UnsupportedEncodingException {

        String seesion = "{\"subIndex\":96,\"trackInfo\":{\"parentdrawerid\":\"34441\"},\"spmA\":\"a2h05\",\"level\":2,\"spmC\":\"drawer3\",\"spmB\":\"8165803_SHAIXUAN_ALL\",\"index\":1,\"pageName\":\"page_channelmain_SHAIXUAN_ALL\",\"scene\":\"search_component_paging\",\"scmB\":\"manual\",\"path\":\"EP782932\",\"scmA\":\"20140719\",\"scmC\":\"34441\",\"from\":\"SHAIXUAN\",\"id\":227939,\"category\":\"电视剧\"}";
        net.sf.json.JSONObject sessionJSON = net.sf.json.JSONObject.fromObject(seesion);
        sessionJSON.put("subIndex", (page - 1) * pageSize);
        sessionJSON.put("category", category);
        String url = String.format("https://www.youku.com/category/data?session=%s&params=%s&pageNo=%s", URLEncoder.encode(sessionJSON.toString(), "UTF-8"), URLEncoder.encode("{\"type\":\"" + category + "\"}", "UTF-8"), page);
        Map<String,String> headers=new HashMap<>();
        headers.put("Accept","text/javascript, text/html, application/xml, text/xml, */*");
        headers.put("Bx-V","2.5.11");
        headers.put("Content-Type","application/x-www-form-urlencoded");
        headers.put("Referer","https://www.youku.com/channel/webtv/list?");

        headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
        headers.put("Sec-Ch-Ua","\"Chromium\";v=\"122\", \"Not(A:Brand\";v=\"24\", \"Google Chrome\";v=\"122\"");
        headers.put("Sec-Ch-Ua-Mobile","?0");
        headers.put("Sec-Ch-Ua-Platform","\"Windows\"");
        headers.put("Sec-Fetch-Dest","empty");
        headers.put("Sec-Fetch-Mode","cors");
        headers.put("Sec-Fetch-Site","same-origin");

        String result = HttpUtil.get(url,new HashMap<>(),headers);
        net.sf.json.JSONObject resultJSON =    net.sf.json.JSONObject.fromObject(result);
        List<YouKuShowSimple> list =new ArrayList<>();
        if(resultJSON.optBoolean("success")){
            net.sf.json.JSONArray array = resultJSON.optJSONObject("data").optJSONObject("filterData").optJSONArray("listData");
            for(int i=0;i<array.size();i++){
                net.sf.json.JSONObject item =  array.optJSONObject(i);
                YouKuShowSimple show=new YouKuShowSimple();
                show.setPoster(item.optString("img"));
                show.setName(item.optString("title"));
                show.setLink(item.optString("videoLink"));
                URI uri=URI.create(show.getLink());
                show.setId(uri.getQuery().replace("s=","").trim());
                list.add(show);
            }
        }
        return new ListResultDTO(list, 100);
    }

    public static ListResultDTO getVideoList(String showId, int page, int pageSize) {
        Map<String, String> params = new HashMap<>();
        params.put("show_id", showId);
@@ -117,11 +168,14 @@
    }


    public static void main(String[] args) {
    public static void main(String[] args) throws  Exception{

        getSearchRank(null,10);
        getSearchRank("电视剧",10);
//        ListResultDTO dto = getShowListByCategory("电视剧",1,20);
//        System.out.printf(dto.toString());
//        getHotShowListByCategory("电视剧",1,24);

//        YouKuShowDetail detail = getShowDetail("1e61efbfbdefbfbd04ef");
//        YouKuShowDetail detail = getShowDetail("ffaac0420f0042b9b1e1");
//        System.out.println(detail);
//        getVideoList("0a17e614d7e311e68ce4", 1, 20);