增加拼多多与京东商品爬取JS

admin

2019-08-08 298251cfb5014e18fe25191f65bb65a0fca32904

增加拼多多与京东商品爬取JS

 fanli/src/main/java/com/yeshi/fanli/controller/client/v2/ConfigControllerV2.java

@@ -33,7 +33,6 @@
import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.ThreadUtil;

import com.yeshi.fanli.util.VersionUtil;

import com.yeshi.fanli.util.jd.JDApiUtil;

import com.yeshi.fanli.util.jd.JDUtil;



import net.sf.json.JSONObject;

@@ -261,7 +260,8 @@


            data.put("baichuan", false);// 不需要采用阿里百川的方式加载webview



        } else if (url != null && ((url.contains("yangkeduo.com") && !url.contains("yangkeduo.com/goods.html?"))

        } else if (url != null && ((url.contains("yangkeduo.com") && !url.contains("yangkeduo.com/goods.html?")

                && !url.contains("yangkeduo.com/duo_coupon_landing.html?"))

                || (url.startsWith("https://p.pinduoduo.com/")))) {

            data.put("baichuan", false);// 不需要采用阿里百川的方式加载webview

            data.put("goodsDetail", true);// 不需要拦截商品详情


 fanli/src/main/java/com/yeshi/fanli/service/impl/order/CommonOrderServiceImpl.java

@@ -330,7 +330,7 @@


        // 是否存在奖励券

        boolean hasRewardCoupon = userSystemCouponService.hasRewardCoupon(uid);

		


        // 2019.8.1开始返回维权信息

        Date august = null;

        try {

@@ -390,7 +390,7 @@
            if (hongBao == null) {

                hongBao = new BigDecimal(0);

            }

			


            if (CommonOrder.STATE_FK == orderState) {

                orderStateContent = "已付款";

            } else if (CommonOrder.STATE_JS == orderState) {

@@ -422,7 +422,7 @@
                            isPart = true;

                        }

                    }

					


                    if (!VersionUtil.greaterThan_1_6_0(acceptData.getPlatform(), acceptData.getVersion())) {

                        if (isPart) {

                            hongBaoState = HongBaoV2.STATE_BUFENSHIXIAO; // 部分失效

@@ -438,11 +438,14 @@
                            if (isPart) {

                                // 维权失效金额=维权的金额*获得的红包/结算金额

                                BigDecimal settlement = order.getSettlement();

                                BigDecimal wqHongbao = MoneyBigDecimalUtil.div(MoneyBigDecimalUtil.mul(hongBao, settlement), settlement);

                                weiQuanInfo.setWqHongBao("维权：-¥" + wqHongbao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                BigDecimal wqHongbao = MoneyBigDecimalUtil

                                        .div(MoneyBigDecimalUtil.mul(hongBao, settlement), settlement);

                                weiQuanInfo.setWqHongBao(

                                        "维权：-¥" + wqHongbao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                hongBao = MoneyBigDecimalUtil.sub(hongBao, wqHongbao);

                            } else {

                                weiQuanInfo.setWqHongBao("维权：-¥" + hongBao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                weiQuanInfo

                                        .setWqHongBao("维权：-¥" + hongBao.setScale(2, BigDecimal.ROUND_DOWN).toString());

                                hongBao = new BigDecimal(0);

                            }

                            order.setWeiQuanInfo(weiQuanInfo);

@@ -533,7 +536,6 @@


            hongbaoInfo = hongBaoState_Str + hongbaoInfo;



			
            Map<String, String> hongBaoMap = new HashMap<String, String>();

            hongBaoMap.put("content", hongbaoInfo + " ¥" + hongBao.setScale(2, BigDecimal.ROUND_DOWN).toString());

            hongBaoMap.put("fontColor", hongbaoInfoFontColor);

@@ -905,6 +907,7 @@
        } else {// 修改

            // 已经结算，已经失效，状态未改变的订单不处理

            if (oldCommonOrder.getState() == CommonOrder.STATE_JS || oldCommonOrder.getState() == CommonOrder.STATE_SX

                    || oldCommonOrder.getState() == CommonOrder.STATE_WQ

                    || (oldCommonOrder.getState().intValue() == commonOrder.getState())) {

                return oldCommonOrder;

            }


 fanli/src/main/java/com/yeshi/fanli/service/impl/taobao/dataoke/DaTaoKeGoodsDetailV2ServiceImpl.java

@@ -40,24 +40,37 @@
    @Override

    public void startSyncGoods() {

        LogHelper.test("大淘客同步开始");

        DaTaoKeGoodsResult result = DaTaoKeApiUtil.listAll("1");

        int page = 0;

        while (result.getGoodsList() != null && result.getGoodsList().size() > 0) {

            System.out.println(page++);

            for (DaTaoKeDetailV2 v2 : result.getGoodsList())

            {

                try {

                    daTaoKeGoodsDetailV2Dao.saveWithCommision(v2);

                } catch (Exception e) {



        int totalCount = 0;

        try {



            DaTaoKeGoodsResult result = DaTaoKeApiUtil.listAll("1");

            int page = 0;



            while (result.getGoodsList() != null && result.getGoodsList().size() > 0) {

                System.out.println(page++);

                totalCount += result.getGoodsList().size();

                for (DaTaoKeDetailV2 v2 : result.getGoodsList()) {

                    try {

                        daTaoKeGoodsDetailV2Dao.saveWithCommision(v2);

                    } catch (Exception e) {

                    }

                }

				


                try {

                    result = DaTaoKeApiUtil.listAll(result.getPageId());

                } catch (Exception e) {

                    result = DaTaoKeApiUtil.listAll(result.getPageId());

                }



                try {

                    Thread.sleep(10);

                } catch (InterruptedException e) {

                    e.printStackTrace();

                }

            }

            result = DaTaoKeApiUtil.listAll(result.getPageId());

            try {

                Thread.sleep(10);

            } catch (InterruptedException e) {

                e.printStackTrace();

            }

        } finally {

            LogHelper.test("大淘客新版商品同步数量:" + totalCount);

        }

    }



@@ -103,7 +116,7 @@
        String startTime = null;

        DaTaoKeDetailV2 latest = daTaoKeGoodsDetailV2Dao.selectLatest();

        if (latest != null) {

            startTime =TimeUtil.getGernalTime(latest.getCreateTime().getTime(),"yyyy-MM-dd HH:mm:ss");

            startTime = TimeUtil.getGernalTime(latest.getCreateTime().getTime(), "yyyy-MM-dd HH:mm:ss");

        }

        DaTaoKeGoodsResult result = DaTaoKeApiUtil.getNewGoodsList(null, startTime, null);

        while (!StringUtil.isNullOrEmpty(result.getPageId())) {

@@ -199,8 +212,7 @@
    @Override

    public List<DaTaoKeDetailV2> listPreShareHotGoods(int page) {

        Map<Long, DaTaoKeDetailV2> map = new HashMap<>();

        List<DaTaoKeDetailV2> list = daTaoKeGoodsDetailV2Dao.listPreShareHotGoods((page - 1) *100,

                100, "monthSales");

        List<DaTaoKeDetailV2> list = daTaoKeGoodsDetailV2Dao.listPreShareHotGoods((page - 1) * 100, 100, "monthSales");

        System.out.println(list);

        for (DaTaoKeDetailV2 v2 : list) {

            map.put(v2.getId(), v2);

@@ -235,7 +247,7 @@
            for (DaTaoKeDetailV2 v2 : result.getGoodsList()) {

                BigDecimal commision = MoneyBigDecimalUtil

                        .div(MoneyBigDecimalUtil.mul(v2.getActualPrice(), v2.getCommissionRate()), new BigDecimal(100));




                if (commision.multiply(new BigDecimal("0.7")).compareTo(new BigDecimal("1.01")) >= 0

                        && v2.getActualPrice().compareTo(new BigDecimal(30)) < 0 && v2.getMonthSales() >= 5000) {

                    Date time = new Date(TimeUtil.convertToTimeTemp(v2.getCouponEndTime(), "yyyy-MM-dd HH:mm:ss"));

@@ -252,7 +264,7 @@
                    }

                    if (time.getTime() >= targetTime.getTime())

                        goodsList.add(v2);

                }
                }

            }

            return goodsList;

        } else


 fanli/src/main/java/com/yeshi/fanli/util/jd/JDUtil.java

@@ -1,6 +1,9 @@
package com.yeshi.fanli.util.jd;



import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.UnsupportedEncodingException;

import java.math.BigDecimal;

import java.net.URLEncoder;

@@ -524,26 +527,39 @@


        return null;

    }

	
    public static String getJDGoodsId(String url) {

        try {

            if (url.startsWith("https://item.m.jd.com/product/") || url.startsWith("http://item.m.jd.com/product/")

                    || url.startsWith("https://item.jd.com/") || url.startsWith("http://item.jd.com/")) {

                String preUrl = url.split("\\?")[0];

                String index = preUrl.split("/")[preUrl.split("/").length - 1];

                index = index.split("\\.")[0];

                return index.trim();

            } else if (url.startsWith("https://item.m.jd.com/ware/view.action?")) {

                String preUrl = url.substring(url.indexOf("?") + 1, url.length());

                String[] sts = preUrl.split("&");

                for (String st : sts) {

                    if (st.startsWith("wareId=")) {

                        return st.split("=")[1].trim();

                    }

                }

            }



    public static String getJDGoodsJS() {

        InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("jdGoods.js");

        StringBuilder sb = new StringBuilder();

        String line;

        BufferedReader br = new BufferedReader(new InputStreamReader(input));

        try {

            while ((line = br.readLine()) != null) {

                sb.append(line);

            }

        } catch (IOException e) {

            e.printStackTrace();

        }

        return sb.toString();

    }



    static String jdGoodsJs = null;



    public static String getJDGoodsId(String url) {

        if (jdGoodsJs == null)

            jdGoodsJs = getJDGoodsJS();

        ScriptEngineManager manager = new ScriptEngineManager();

        ScriptEngine engine = manager.getEngineByName("javascript");

        try {

            engine.eval(jdGoodsJs);

            if (engine instanceof Invocable) {

                Invocable in = (Invocable) engine;

                String goodsId = in.invokeFunction("getGoodsId", url).toString();

                if (!StringUtil.isNullOrEmpty(goodsId))

                    return goodsId.trim();

            }

        } catch (Exception e) {

            e.printStackTrace();

        }

        return null;

    }


 fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java

@@ -1,5 +1,9 @@
package com.yeshi.fanli.util.pinduoduo;



import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.UnsupportedEncodingException;

import java.math.BigDecimal;

import java.net.URLEncoder;

@@ -24,6 +28,7 @@
import com.yeshi.fanli.log.LogHelper;

import com.yeshi.fanli.util.MoneyBigDecimalUtil;

import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.jd.JDUtil;



import net.sf.json.JSONArray;

import net.sf.json.JSONObject;

@@ -145,7 +150,7 @@
        try {

            client.executeMethod(pm);

            String result = pm.getResponseBodyAsString();

            LogHelper.test("拼多多猜你喜欢:"+result);

            LogHelper.test("拼多多猜你喜欢:" + result);

            JSONObject json = JSONObject.fromObject(result);

            JSONArray array = json.optJSONArray("data");

            if (array != null) {

@@ -348,8 +353,7 @@
        }

        return null;

    }

	
	


    /**

     * 多多进宝爬取数据-品牌好货

     * 

@@ -359,8 +363,7 @@
    public static PDDGoodsResult getTodaySaleGoods() {

        JSONObject params = new JSONObject();

        params.put("type", 1);

        params.put("sortType",3);

		
        params.put("sortType", 3);



        HttpClient client = new HttpClient();

        PostMethod pm = new PostMethod("https://jinbao.pinduoduo.com/network/api/common/queryTopGoodsList");

@@ -406,7 +409,6 @@
        return goodsResult;

    }





    private static PDDGoodsDetail parseTodaySaleGoods(JSONObject json) {

        PDDGoodsDetail goods = new PDDGoodsDetail();

        goods.setMallName(json.optString("mallName"));

@@ -419,7 +421,7 @@
        if (json.optString("mallId") != null) {

            goods.setMallId(json.getLong("mallId"));

        }

		


        if (json.optString("goodsImageUrl") != null) {

            goods.setGoodsImageUrl(json.optString("goodsImageUrl"));

        }

@@ -431,7 +433,7 @@
        if (json.optString("minGroupPrice") != null) {

            goods.setMinGroupPrice(json.getLong("minGroupPrice"));

        }

		


        if (json.optString("minNormalPrice") != null) {

            goods.setMinNormalPrice(json.getLong("minNormalPrice"));

        }

@@ -477,23 +479,41 @@


        return goods;

    }

	
    public static String getPDDGoodsId(String url) {



    public static String getJDGoodsJS() {

        InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("pddGoods.js");

        StringBuilder sb = new StringBuilder();

        String line;

        BufferedReader br = new BufferedReader(new InputStreamReader(input));

        try {

            if (url.startsWith("https://mobile.yangkeduo.com/goods.html?")

                    || url.startsWith("http://mobile.yangkeduo.com/goods.html?")||url.contains("yangkeduo.com/duo_coupon_landing.html?")) {

                String preUrl = url.substring(url.indexOf("?") + 1, url.length());

                String[] sts = preUrl.split("&");

                for (String st : sts) {

                    if (st.startsWith("goods_id=")) {

                        return st.split("=")[1].trim();

                    }

                }

            while ((line = br.readLine()) != null) {

                sb.append(line);

            }

        } catch (IOException e) {

            e.printStackTrace();

        }

        return sb.toString();

    }



    static String jdGoodsJs = null;



    public static String getPDDGoodsId(String url) {

        if (jdGoodsJs == null)

            jdGoodsJs = getJDGoodsJS();

        ScriptEngineManager manager = new ScriptEngineManager();

        ScriptEngine engine = manager.getEngineByName("javascript");

        try {

            engine.eval(jdGoodsJs);

            if (engine instanceof Invocable) {

                Invocable in = (Invocable) engine;

                String goodsId = in.invokeFunction("getGoodsId", url).toString();

                if (!StringUtil.isNullOrEmpty(goodsId))

                    return goodsId.trim();

            }

        } catch (Exception e) {

            e.printStackTrace();

        }

        return null;

    }

	
	


}


 fanli/src/main/resource/jdGoods.js

New file
@@ -0,0 +1,17 @@
function getGoodsId(url) {

    if (url.indexOf('://item.m.jd.com/product/') > 0 || url.indexOf('://item.jd.com/') > 0) {

        var preUrl = url.split("?")[0];

        var index = preUrl.split("/")[preUrl.split("/").length - 1];

        index = index.split(".")[0];

        return index.replace(" ", "");

    } else if (url.indexOf('://item.m.jd.com/ware/view.action?') > 0) {

        var preUrl = url.substr(url.indexOf("?") + 1, url.length);

        var sts = preUrl.split("&");

        for (var i = 0; i < sts.length; i++) {

            if (st.indexOf("wareId=") == 0) {

                return st.split("=")[1].replace(" ", "");

            }

        }

    }

    return null;

}

 fanli/src/main/resource/pddGoods.js

New file
@@ -0,0 +1,12 @@
function getGoodsId(url) {

    if (url.indexOf('://mobile.yangkeduo.com/goods.html?') > 0 || url.indexOf('yangkeduo.com/duo_coupon_landing.html?') >

        0 || url.indexOf('://mobile.yangkeduo.com/goods2.html?') > 0) {

        var preUrl = url.substr(url.indexOf("?") + 1, url.length());

        var sts = preUrl.split("&");

        for (var i = 0; i < sts.length; i++) {

            if (sts[i].indexOf("goods_id=") == 0) {

                return sts[i].split("=")[1];

            }

        }

    }

}