Merge remote-tracking branch 'origin/div' into div

admin

2020-04-13 dd5b15229cb15459fa7c31ccea77dac28cbfafbd

 fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java

@@ -1,16 +1,25 @@
package com.yeshi.fanli.util.pinduoduo;



import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.UnsupportedEncodingException;

import java.math.BigDecimal;

import java.net.URLEncoder;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Matcher;

import java.util.regex.Pattern;



import javax.script.Invocable;

import javax.script.ScriptEngine;

import javax.script.ScriptEngineManager;

import javax.script.ScriptException;



import org.apache.commons.httpclient.Header;

import org.apache.commons.httpclient.HttpClient;

import org.apache.commons.httpclient.HttpException;

import org.apache.commons.httpclient.methods.PostMethod;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

@@ -21,8 +30,10 @@
import com.yeshi.fanli.dto.pdd.PDDGoodsDetail;

import com.yeshi.fanli.dto.pdd.PDDGoodsResult;

import com.yeshi.fanli.dto.pdd.PDDSearchFilter;

import com.yeshi.fanli.log.LogHelper;

import com.yeshi.fanli.util.MoneyBigDecimalUtil;

import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.jd.JDUtil;



import net.sf.json.JSONArray;

import net.sf.json.JSONObject;

@@ -144,6 +155,7 @@
      try {

         client.executeMethod(pm);

         String result = pm.getResponseBodyAsString();

         LogHelper.test("拼多多猜你喜欢:" + result);

         JSONObject json = JSONObject.fromObject(result);

         JSONArray array = json.optJSONArray("data");

         if (array != null) {

@@ -347,4 +359,258 @@
      return null;

   }



   /**

    * 多多进宝爬取数据-品牌好货

    * 
    * @param sf

    * @return

    */

   public static PDDGoodsResult getTodaySaleGoods() {

      JSONObject params = new JSONObject();

      params.put("type", 1);

      params.put("sortType", 3);



      HttpClient client = new HttpClient();

      PostMethod pm = new PostMethod("https://jinbao.pinduoduo.com/network/api/common/queryTopGoodsList");

      pm.setRequestHeader("User-Agent",

            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3730.400 QQBrowser/10.5.3805.400");

      pm.setRequestHeader("Content-Type", "application/json;charset=UTF-8");

      pm.setRequestHeader("Referer", "https://jinbao.pinduoduo.com/promotion/hot-promotion");

      pm.setRequestBody(params.toString());



      PDDGoodsResult goodsResult = null;



      try {

         client.executeMethod(pm);

         String result = pm.getResponseBodyAsString();

         JSONObject json = JSONObject.fromObject(result);

         Boolean code = json.optBoolean("success");

         if (code != null && code) {

            JSONObject root = json.optJSONObject("result");

            if (root == null) {

               return null;

            }

            JSONArray array = root.optJSONArray("list");

            if (array == null) {

               return null;

            }



            List<PDDGoodsDetail> goodsList = new ArrayList<PDDGoodsDetail>();

            for (int i = 0; i < array.size(); i++) {

               PDDGoodsDetail parseGoods = parseTodaySaleGoods(array.getJSONObject(i));

               if (parseGoods != null) {

                  goodsList.add(parseGoods);

               }

            }

            int totalCount = root.optInt("total");



            goodsResult = new PDDGoodsResult();

            goodsResult.setGoodsList(goodsList);

            goodsResult.setTotalCount(totalCount);

         }

      } catch (Exception e) {

         e.printStackTrace();

      }



      return goodsResult;

   }



   private static PDDGoodsDetail parseTodaySaleGoods(JSONObject json) {

      PDDGoodsDetail goods = new PDDGoodsDetail();

      goods.setMallName(json.optString("mallName"));

      goods.setMerchantType(json.optInt("merchantType"));

      goods.setGoodsId(json.optLong("goodsId"));

      goods.setGoodsName(json.optString("goodsName"));

      goods.setGoodsDesc(json.optString("goodsDesc"));

      goods.setSalesTip(json.optString("salesTip"));



      if (json.optString("mallId") != null) {

         goods.setMallId(json.getLong("mallId"));

      }



      if (json.optString("goodsImageUrl") != null) {

         goods.setGoodsImageUrl(json.optString("goodsImageUrl"));

      }



      if (json.optString("goodsThumbnailUrl") != null) {

         goods.setGoodsThumbnailUrl(json.optString("goodsThumbnailUrl"));

      }



      if (json.optString("minGroupPrice") != null) {

         goods.setMinGroupPrice(json.getLong("minGroupPrice"));

      }



      if (json.optString("minNormalPrice") != null) {

         goods.setMinNormalPrice(json.getLong("minNormalPrice"));

      }



      if (json.optString("categoryId") != null) {

         goods.setCategoryId(json.getLong("categoryId"));

      }



      goods.setCategoryName(json.optString("categoryName"));

      goods.setHasCoupon(json.optBoolean("hasCoupon"));



      if (json.optString("couponMinOrderAmount") != null) {

         goods.setCouponMinOrderAmount(json.getLong("couponMinOrderAmount"));

      }



      if (json.optString("couponDiscount") != null) {

         goods.setCouponDiscount(json.getLong("couponDiscount"));

      }



      if (json.optString("couponTotalQuantity") != null) {

         goods.setCouponTotalQuantity(json.getLong("couponTotalQuantity"));

      }



      if (json.optString("couponRemainQuantity") != null) {

         goods.setCouponRemainQuantity(json.getLong("couponRemainQuantity"));

      }



      if (json.optString("couponStartTime") != null) {

         goods.setCouponStartTime(json.getLong("couponStartTime"));

      }



      if (json.optString("couponEndTime") != null) {

         goods.setCouponEndTime(json.getLong("couponEndTime"));

      }



      if (json.optString("promotionRate") != null) {

         goods.setPromotionRate(json.getLong("promotionRate"));

      }



      if (json.optString("optId") != null) {

         goods.setOptId(json.getLong("optId"));

      }



      return goods;

   }



   public static String getJDGoodsJS() {

      InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("pddGoods.js");

      StringBuilder sb = new StringBuilder();

      String line;

      BufferedReader br = new BufferedReader(new InputStreamReader(input));

      try {

         while ((line = br.readLine()) != null) {

            sb.append(line);

         }

      } catch (IOException e) {

         e.printStackTrace();

      }

      return sb.toString();

   }



   static String jdGoodsJs = null;

   static ScriptEngine engine = null;

   static {

      if (jdGoodsJs == null)

         jdGoodsJs = getJDGoodsJS();

      ScriptEngineManager manager = new ScriptEngineManager();

      engine = manager.getEngineByName("javascript");

      try {

         engine.eval(jdGoodsJs);

      } catch (ScriptException e) {

         e.printStackTrace();

      }



   }



   public static String getPDDGoodsId(String url) {

      if (url == null)

         return null;

      String link = url;

      if (link.contains("://p.pinduoduo.com/")) {// 拼多多的短链

         HttpClient client = new HttpClient();

         client.getHttpConnectionManager().getParams().setConnectionTimeout(5000);

         PostMethod pm = new PostMethod(link);

         try {

            client.executeMethod(pm);

            Header location = pm.getResponseHeader("Location");

            if (location != null)

               link = location.getValue();

         } catch (HttpException e) {

            e.printStackTrace();

         } catch (IOException e) {

            e.printStackTrace();

         }

      }



      try {

         if (engine instanceof Invocable) {

            Invocable in = (Invocable) engine;

            Object goodsId = in.invokeFunction("getGoodsId", link);

            if (goodsId != null)

               return goodsId.toString().trim();

         }

      } catch (Exception e) {

         e.printStackTrace();

      }

      return null;

   }



   public static PDDGoodsDetail getPDDGoodsInfo(String id) {

      try {

         Document doc = Jsoup.connect("http://yangkeduo.com/goods.html?goods_id=" + id)

               .userAgent(

                     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")

               .get();

         Elements els = doc.getElementsByTag("script");

         for (int i = 0; i < els.size(); i++) {

            if (els.get(i).html().contains("window.rawData")) {

               String dataJS = els.get(i).html().replace("window.", "var ");

               dataJS += "function getData(){return JSON.stringify(rawData);}";



               ScriptEngineManager manager = new ScriptEngineManager();

               ScriptEngine engine = manager.getEngineByName("javascript");

               try {

                  engine.eval(dataJS);

                  if (engine instanceof Invocable) {

                     Invocable in = (Invocable) engine;

                     String jsonStr = in.invokeFunction("getData").toString();

                     JSONObject json = JSONObject.fromObject(jsonStr);



                     JSONObject goods = json.optJSONObject("store").optJSONObject("initDataObj")

                           .optJSONObject("goods");



                     String imageData = goods.optString("viewImageData");

                     imageData = imageData.replace("[", "").replace("]", "");

                     String[] images = imageData.split(",");



                     String image = images[0].trim();

                     if (image.startsWith("\""))

                        image = image.substring(1, image.length() - 1);

                     if (image.endsWith("\""))

                        image = image.substring(0, image.length() - 2);



                     PDDGoodsDetail goodsDetail = new PDDGoodsDetail();

                     goodsDetail.setGoodsImageUrl("http:" + image);

                     goodsDetail.setGoodsThumbnailUrl("http:" + image);

                     goodsDetail.setGoodsName(goods.optString("goodsName"));

                     goodsDetail.setGoodsId(goods.optLong("goodsID"));

                     return goodsDetail;

                  }

               } catch (Exception e) {

                  e.printStackTrace();

               }

               break;

            }

         }

      } catch (Exception e1) {

         e1.printStackTrace();

      }

      return null;

   }



   public static List<String> getPDDShortLinksFromText(String text) {

      String regex = "(https://p\\.pinduoduo\\.com/)[0-9A-Za-z]{1,20}";

      Pattern pattern = Pattern.compile(regex);

      Matcher m = pattern.matcher(text);

      List<String> urlList = new ArrayList<>();

      while (m.find()) {

         urlList.add(m.group());

      }

      return urlList;

   }



}