增加拼多多与京东商品爬取JS

admin

2019-08-08 298251cfb5014e18fe25191f65bb65a0fca32904

增加拼多多与京东商品爬取JS

 fanli/src/main/java/com/yeshi/fanli/controller/client/v2/ConfigControllerV2.java

@@ -33,7 +33,6 @@
import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.ThreadUtil;

import com.yeshi.fanli.util.VersionUtil;

import com.yeshi.fanli.util.jd.JDApiUtil;

import com.yeshi.fanli.util.jd.JDUtil;



import net.sf.json.JSONObject;

@@ -261,7 +260,8 @@


            data.put("baichuan", false);// 不需要采用阿里百川的方式加载webview



        } else if (url != null && ((url.contains("yangkeduo.com") && !url.contains("yangkeduo.com/goods.html?"))

        } else if (url != null && ((url.contains("yangkeduo.com") && !url.contains("yangkeduo.com/goods.html?")

                && !url.contains("yangkeduo.com/duo_coupon_landing.html?"))

                || (url.startsWith("https://p.pinduoduo.com/")))) {

            data.put("baichuan", false);// 不需要采用阿里百川的方式加载webview

            data.put("goodsDetail", true);// 不需要拦截商品详情


 fanli/src/main/java/com/yeshi/fanli/service/impl/order/CommonOrderServiceImpl.java

@@ -438,11 +438,14 @@
                            if (isPart) {

                                // 维权失效金额=维权的金额*获得的红包/结算金额

                                BigDecimal settlement = order.getSettlement();

                                BigDecimal wqHongbao = MoneyBigDecimalUtil.div(MoneyBigDecimalUtil.mul(hongBao, settlement), settlement);

                                weiQuanInfo.setWqHongBao("维权：-¥" + wqHongbao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                BigDecimal wqHongbao = MoneyBigDecimalUtil

                                        .div(MoneyBigDecimalUtil.mul(hongBao, settlement), settlement);

                                weiQuanInfo.setWqHongBao(

                                        "维权：-¥" + wqHongbao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                hongBao = MoneyBigDecimalUtil.sub(hongBao, wqHongbao);

                            } else {

                                weiQuanInfo.setWqHongBao("维权：-¥" + hongBao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                weiQuanInfo

                                        .setWqHongBao("维权：-¥" + hongBao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                hongBao = new BigDecimal(0);

                            }

                            order.setWeiQuanInfo(weiQuanInfo);

@@ -532,7 +535,6 @@
            order.setAccountState(stateMap);



            hongbaoInfo = hongBaoState_Str + hongbaoInfo;



            

            Map<String, String> hongBaoMap = new HashMap<String, String>();

            hongBaoMap.put("content", hongbaoInfo + " ¥" + hongBao.setScale(2, BigDecimal.ROUND_DOWN).toString());

@@ -905,6 +907,7 @@
        } else {// 修改

            // 已经结算，已经失效，状态未改变的订单不处理

            if (oldCommonOrder.getState() == CommonOrder.STATE_JS || oldCommonOrder.getState() == CommonOrder.STATE_SX

                    || oldCommonOrder.getState() == CommonOrder.STATE_WQ

                    || (oldCommonOrder.getState().intValue() == commonOrder.getState())) {

                return oldCommonOrder;

            }


 fanli/src/main/java/com/yeshi/fanli/service/impl/taobao/dataoke/DaTaoKeGoodsDetailV2ServiceImpl.java

@@ -40,24 +40,37 @@
    @Override

    public void startSyncGoods() {

        LogHelper.test("大淘客同步开始");



        int totalCount = 0;

        try {



        DaTaoKeGoodsResult result = DaTaoKeApiUtil.listAll("1");

        int page = 0;



        while (result.getGoodsList() != null && result.getGoodsList().size() > 0) {

            System.out.println(page++);

            for (DaTaoKeDetailV2 v2 : result.getGoodsList())

            {

                totalCount += result.getGoodsList().size();

                for (DaTaoKeDetailV2 v2 : result.getGoodsList()) {

                try {

                    daTaoKeGoodsDetailV2Dao.saveWithCommision(v2);

                } catch (Exception e) {

                }

				
            }



                try {

            result = DaTaoKeApiUtil.listAll(result.getPageId());

                } catch (Exception e) {

                    result = DaTaoKeApiUtil.listAll(result.getPageId());

                }



            try {

                Thread.sleep(10);

            } catch (InterruptedException e) {

                e.printStackTrace();

            }

            }

        } finally {

            LogHelper.test("大淘客新版商品同步数量:" + totalCount);

        }

    }



@@ -199,8 +212,7 @@
    @Override

    public List<DaTaoKeDetailV2> listPreShareHotGoods(int page) {

        Map<Long, DaTaoKeDetailV2> map = new HashMap<>();

        List<DaTaoKeDetailV2> list = daTaoKeGoodsDetailV2Dao.listPreShareHotGoods((page - 1) *100,

                100, "monthSales");

        List<DaTaoKeDetailV2> list = daTaoKeGoodsDetailV2Dao.listPreShareHotGoods((page - 1) * 100, 100, "monthSales");

        System.out.println(list);

        for (DaTaoKeDetailV2 v2 : list) {

            map.put(v2.getId(), v2);


 fanli/src/main/java/com/yeshi/fanli/util/jd/JDUtil.java

@@ -1,6 +1,9 @@
package com.yeshi.fanli.util.jd;



import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.UnsupportedEncodingException;

import java.math.BigDecimal;

import java.net.URLEncoder;

@@ -525,25 +528,38 @@
        return null;

    }

    

    public static String getJDGoodsId(String url) {

    public static String getJDGoodsJS() {

        InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("jdGoods.js");

        StringBuilder sb = new StringBuilder();

        String line;

        BufferedReader br = new BufferedReader(new InputStreamReader(input));

        try {

            if (url.startsWith("https://item.m.jd.com/product/") || url.startsWith("http://item.m.jd.com/product/")

                    || url.startsWith("https://item.jd.com/") || url.startsWith("http://item.jd.com/")) {

                String preUrl = url.split("\\?")[0];

                String index = preUrl.split("/")[preUrl.split("/").length - 1];

                index = index.split("\\.")[0];

                return index.trim();

            } else if (url.startsWith("https://item.m.jd.com/ware/view.action?")) {

                String preUrl = url.substring(url.indexOf("?") + 1, url.length());

                String[] sts = preUrl.split("&");

                for (String st : sts) {

                    if (st.startsWith("wareId=")) {

                        return st.split("=")[1].trim();

            while ((line = br.readLine()) != null) {

                sb.append(line);

                    }

        } catch (IOException e) {

            e.printStackTrace();

                }

        return sb.toString();

            }



    static String jdGoodsJs = null;



    public static String getJDGoodsId(String url) {

        if (jdGoodsJs == null)

            jdGoodsJs = getJDGoodsJS();

        ScriptEngineManager manager = new ScriptEngineManager();

        ScriptEngine engine = manager.getEngineByName("javascript");

        try {

            engine.eval(jdGoodsJs);

            if (engine instanceof Invocable) {

                Invocable in = (Invocable) engine;

                String goodsId = in.invokeFunction("getGoodsId", url).toString();

                if (!StringUtil.isNullOrEmpty(goodsId))

                    return goodsId.trim();

            }

        } catch (Exception e) {

            e.printStackTrace();

        }

        return null;

    }


 fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java

@@ -1,5 +1,9 @@
package com.yeshi.fanli.util.pinduoduo;



import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.UnsupportedEncodingException;

import java.math.BigDecimal;

import java.net.URLEncoder;

@@ -24,6 +28,7 @@
import com.yeshi.fanli.log.LogHelper;

import com.yeshi.fanli.util.MoneyBigDecimalUtil;

import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.jd.JDUtil;



import net.sf.json.JSONArray;

import net.sf.json.JSONObject;

@@ -349,7 +354,6 @@
        return null;

    }

    

	
    /**

     * 多多进宝爬取数据-品牌好货

     * 

@@ -360,7 +364,6 @@
        JSONObject params = new JSONObject();

        params.put("type", 1);

        params.put("sortType",3);

		


        HttpClient client = new HttpClient();

        PostMethod pm = new PostMethod("https://jinbao.pinduoduo.com/network/api/common/queryTopGoodsList");

@@ -405,7 +408,6 @@


        return goodsResult;

    }





    private static PDDGoodsDetail parseTodaySaleGoods(JSONObject json) {

        PDDGoodsDetail goods = new PDDGoodsDetail();

@@ -478,22 +480,40 @@
        return goods;

    }

    

    public static String getPDDGoodsId(String url) {

    public static String getJDGoodsJS() {

        InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("pddGoods.js");

        StringBuilder sb = new StringBuilder();

        String line;

        BufferedReader br = new BufferedReader(new InputStreamReader(input));

        try {

            if (url.startsWith("https://mobile.yangkeduo.com/goods.html?")

                    || url.startsWith("http://mobile.yangkeduo.com/goods.html?")||url.contains("yangkeduo.com/duo_coupon_landing.html?")) {

                String preUrl = url.substring(url.indexOf("?") + 1, url.length());

                String[] sts = preUrl.split("&");

                for (String st : sts) {

                    if (st.startsWith("goods_id=")) {

                        return st.split("=")[1].trim();

            while ((line = br.readLine()) != null) {

                sb.append(line);

                    }

        } catch (IOException e) {

            e.printStackTrace();

                }

        return sb.toString();

    }



    static String jdGoodsJs = null;



    public static String getPDDGoodsId(String url) {

        if (jdGoodsJs == null)

            jdGoodsJs = getJDGoodsJS();

        ScriptEngineManager manager = new ScriptEngineManager();

        ScriptEngine engine = manager.getEngineByName("javascript");

        try {

            engine.eval(jdGoodsJs);

            if (engine instanceof Invocable) {

                Invocable in = (Invocable) engine;

                String goodsId = in.invokeFunction("getGoodsId", url).toString();

                if (!StringUtil.isNullOrEmpty(goodsId))

                    return goodsId.trim();

            }

        } catch (Exception e) {

            e.printStackTrace();

        }

        return null;

    }

	
    

}


 fanli/src/main/resource/jdGoods.js

New file
@@ -0,0 +1,17 @@
function getGoodsId(url) {

    if (url.indexOf('://item.m.jd.com/product/') > 0 || url.indexOf('://item.jd.com/') > 0) {

        var preUrl = url.split("?")[0];

        var index = preUrl.split("/")[preUrl.split("/").length - 1];

        index = index.split(".")[0];

        return index.replace(" ", "");

    } else if (url.indexOf('://item.m.jd.com/ware/view.action?') > 0) {

        var preUrl = url.substr(url.indexOf("?") + 1, url.length);

        var sts = preUrl.split("&");

        for (var i = 0; i < sts.length; i++) {

            if (st.indexOf("wareId=") == 0) {

                return st.split("=")[1].replace(" ", "");

            }

        }

    }

    return null;

}

 fanli/src/main/resource/pddGoods.js

New file
@@ -0,0 +1,12 @@
function getGoodsId(url) {

    if (url.indexOf('://mobile.yangkeduo.com/goods.html?') > 0 || url.indexOf('yangkeduo.com/duo_coupon_landing.html?') >

        0 || url.indexOf('://mobile.yangkeduo.com/goods2.html?') > 0) {

        var preUrl = url.substr(url.indexOf("?") + 1, url.length());

        var sts = preUrl.split("&");

        for (var i = 0; i < sts.length; i++) {

            if (sts[i].indexOf("goods_id=") == 0) {

                return sts[i].split("=")[1];

            }

        }

    }

}