Merge remote-tracking branch 'origin/div' into div

admin

2020-04-13 dd5b15229cb15459fa7c31ccea77dac28cbfafbd

 fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java

@@ -1,16 +1,25 @@
package com.yeshi.fanli.util.pinduoduo;



import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.UnsupportedEncodingException;

import java.math.BigDecimal;

import java.net.URLEncoder;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Matcher;

import java.util.regex.Pattern;



import javax.script.Invocable;

import javax.script.ScriptEngine;

import javax.script.ScriptEngineManager;

import javax.script.ScriptException;



import org.apache.commons.httpclient.Header;

import org.apache.commons.httpclient.HttpClient;

import org.apache.commons.httpclient.HttpException;

import org.apache.commons.httpclient.methods.PostMethod;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

@@ -24,6 +33,7 @@
import com.yeshi.fanli.log.LogHelper;

import com.yeshi.fanli.util.MoneyBigDecimalUtil;

import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.jd.JDUtil;



import net.sf.json.JSONArray;

import net.sf.json.JSONObject;

@@ -145,7 +155,7 @@
      try {

         client.executeMethod(pm);

         String result = pm.getResponseBodyAsString();

         LogHelper.test("拼多多猜你喜欢:"+result);

         LogHelper.test("拼多多猜你喜欢:" + result);

         JSONObject json = JSONObject.fromObject(result);

         JSONArray array = json.optJSONArray("data");

         if (array != null) {

@@ -348,8 +358,7 @@
      }

      return null;

   }

	
	


   /**

    * 多多进宝爬取数据-品牌好货

    * 

@@ -359,12 +368,12 @@
   public static PDDGoodsResult getTodaySaleGoods() {

      JSONObject params = new JSONObject();

      params.put("type", 1);

      params.put("sortType",3);

		
      params.put("sortType", 3);



      HttpClient client = new HttpClient();

      PostMethod pm = new PostMethod("https://jinbao.pinduoduo.com/network/api/common/queryTopGoodsList");

      pm.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko");

      pm.setRequestHeader("User-Agent",

            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3730.400 QQBrowser/10.5.3805.400");

      pm.setRequestHeader("Content-Type", "application/json;charset=UTF-8");

      pm.setRequestHeader("Referer", "https://jinbao.pinduoduo.com/promotion/hot-promotion");

      pm.setRequestBody(params.toString());

@@ -406,7 +415,6 @@
      return goodsResult;

   }





   private static PDDGoodsDetail parseTodaySaleGoods(JSONObject json) {

      PDDGoodsDetail goods = new PDDGoodsDetail();

      goods.setMallName(json.optString("mallName"));

@@ -419,7 +427,7 @@
      if (json.optString("mallId") != null) {

         goods.setMallId(json.getLong("mallId"));

      }

		


      if (json.optString("goodsImageUrl") != null) {

         goods.setGoodsImageUrl(json.optString("goodsImageUrl"));

      }

@@ -431,7 +439,7 @@
      if (json.optString("minGroupPrice") != null) {

         goods.setMinGroupPrice(json.getLong("minGroupPrice"));

      }

		


      if (json.optString("minNormalPrice") != null) {

         goods.setMinNormalPrice(json.getLong("minNormalPrice"));

      }

@@ -477,23 +485,132 @@


      return goods;

   }

	
   public static String getPDDGoodsId(String url) {



   public static String getJDGoodsJS() {

      InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("pddGoods.js");

      StringBuilder sb = new StringBuilder();

      String line;

      BufferedReader br = new BufferedReader(new InputStreamReader(input));

      try {

         if (url.startsWith("https://mobile.yangkeduo.com/goods.html?")

               || url.startsWith("http://mobile.yangkeduo.com/goods.html?")||url.contains("yangkeduo.com/duo_coupon_landing.html?")) {

            String preUrl = url.substring(url.indexOf("?") + 1, url.length());

            String[] sts = preUrl.split("&");

            for (String st : sts) {

               if (st.startsWith("goods_id=")) {

                  return st.split("=")[1].trim();

               }

            }

         while ((line = br.readLine()) != null) {

            sb.append(line);

         }

      } catch (IOException e) {

         e.printStackTrace();

      }

      return sb.toString();

   }



   static String jdGoodsJs = null;

   static ScriptEngine engine = null;

   static {

      if (jdGoodsJs == null)

         jdGoodsJs = getJDGoodsJS();

      ScriptEngineManager manager = new ScriptEngineManager();

      engine = manager.getEngineByName("javascript");

      try {

         engine.eval(jdGoodsJs);

      } catch (ScriptException e) {

         e.printStackTrace();

      }



   }



   public static String getPDDGoodsId(String url) {

      if (url == null)

         return null;

      String link = url;

      if (link.contains("://p.pinduoduo.com/")) {// 拼多多的短链

         HttpClient client = new HttpClient();

         client.getHttpConnectionManager().getParams().setConnectionTimeout(5000);

         PostMethod pm = new PostMethod(link);

         try {

            client.executeMethod(pm);

            Header location = pm.getResponseHeader("Location");

            if (location != null)

               link = location.getValue();

         } catch (HttpException e) {

            e.printStackTrace();

         } catch (IOException e) {

            e.printStackTrace();

         }

      }



      try {

         if (engine instanceof Invocable) {

            Invocable in = (Invocable) engine;

            Object goodsId = in.invokeFunction("getGoodsId", link);

            if (goodsId != null)

               return goodsId.toString().trim();

         }

      } catch (Exception e) {

         e.printStackTrace();

      }

      return null;

   }

	
	


   public static PDDGoodsDetail getPDDGoodsInfo(String id) {

      try {

         Document doc = Jsoup.connect("http://yangkeduo.com/goods.html?goods_id=" + id)

               .userAgent(

                     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")

               .get();

         Elements els = doc.getElementsByTag("script");

         for (int i = 0; i < els.size(); i++) {

            if (els.get(i).html().contains("window.rawData")) {

               String dataJS = els.get(i).html().replace("window.", "var ");

               dataJS += "function getData(){return JSON.stringify(rawData);}";



               ScriptEngineManager manager = new ScriptEngineManager();

               ScriptEngine engine = manager.getEngineByName("javascript");

               try {

                  engine.eval(dataJS);

                  if (engine instanceof Invocable) {

                     Invocable in = (Invocable) engine;

                     String jsonStr = in.invokeFunction("getData").toString();

                     JSONObject json = JSONObject.fromObject(jsonStr);



                     JSONObject goods = json.optJSONObject("store").optJSONObject("initDataObj")

                           .optJSONObject("goods");



                     String imageData = goods.optString("viewImageData");

                     imageData = imageData.replace("[", "").replace("]", "");

                     String[] images = imageData.split(",");



                     String image = images[0].trim();

                     if (image.startsWith("\""))

                        image = image.substring(1, image.length() - 1);

                     if (image.endsWith("\""))

                        image = image.substring(0, image.length() - 2);



                     PDDGoodsDetail goodsDetail = new PDDGoodsDetail();

                     goodsDetail.setGoodsImageUrl("http:" + image);

                     goodsDetail.setGoodsThumbnailUrl("http:" + image);

                     goodsDetail.setGoodsName(goods.optString("goodsName"));

                     goodsDetail.setGoodsId(goods.optLong("goodsID"));

                     return goodsDetail;

                  }

               } catch (Exception e) {

                  e.printStackTrace();

               }

               break;

            }

         }

      } catch (Exception e1) {

         e1.printStackTrace();

      }

      return null;

   }



   public static List<String> getPDDShortLinksFromText(String text) {

      String regex = "(https://p\\.pinduoduo\\.com/)[0-9A-Za-z]{1,20}";

      Pattern pattern = Pattern.compile(regex);

      Matcher m = pattern.matcher(text);

      List<String> urlList = new ArrayList<>();

      while (m.find()) {

         urlList.add(m.group());

      }

      return urlList;

   }



}