From 298251cfb5014e18fe25191f65bb65a0fca32904 Mon Sep 17 00:00:00 2001
From: admin <weikou2014>
Date: 星期四, 08 八月 2019 18:52:47 +0800
Subject: [PATCH] 增加拼多多与京东商品爬取JS
---
fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java | 62 ++++++++++++++++++++----------
1 files changed, 41 insertions(+), 21 deletions(-)
diff --git a/fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java b/fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java
index 6a44fe1..e4fc9c2 100644
--- a/fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java
+++ b/fanli/src/main/java/com/yeshi/fanli/util/pinduoduo/PinDuoDuoUtil.java
@@ -1,5 +1,9 @@
package com.yeshi.fanli.util.pinduoduo;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.net.URLEncoder;
@@ -24,6 +28,7 @@
import com.yeshi.fanli.log.LogHelper;
import com.yeshi.fanli.util.MoneyBigDecimalUtil;
import com.yeshi.fanli.util.StringUtil;
+import com.yeshi.fanli.util.jd.JDUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
@@ -145,7 +150,7 @@
try {
client.executeMethod(pm);
String result = pm.getResponseBodyAsString();
- LogHelper.test("鎷煎澶氱寽浣犲枩娆�:"+result);
+ LogHelper.test("鎷煎澶氱寽浣犲枩娆�:" + result);
JSONObject json = JSONObject.fromObject(result);
JSONArray array = json.optJSONArray("data");
if (array != null) {
@@ -348,8 +353,7 @@
}
return null;
}
-
-
+
/**
* 澶氬杩涘疂鐖彇鏁版嵁-鍝佺墝濂借揣
*
@@ -359,8 +363,7 @@
public static PDDGoodsResult getTodaySaleGoods() {
JSONObject params = new JSONObject();
params.put("type", 1);
- params.put("sortType",3);
-
+ params.put("sortType", 3);
HttpClient client = new HttpClient();
PostMethod pm = new PostMethod("https://jinbao.pinduoduo.com/network/api/common/queryTopGoodsList");
@@ -406,7 +409,6 @@
return goodsResult;
}
-
private static PDDGoodsDetail parseTodaySaleGoods(JSONObject json) {
PDDGoodsDetail goods = new PDDGoodsDetail();
goods.setMallName(json.optString("mallName"));
@@ -419,7 +421,7 @@
if (json.optString("mallId") != null) {
goods.setMallId(json.getLong("mallId"));
}
-
+
if (json.optString("goodsImageUrl") != null) {
goods.setGoodsImageUrl(json.optString("goodsImageUrl"));
}
@@ -431,7 +433,7 @@
if (json.optString("minGroupPrice") != null) {
goods.setMinGroupPrice(json.getLong("minGroupPrice"));
}
-
+
if (json.optString("minNormalPrice") != null) {
goods.setMinNormalPrice(json.getLong("minNormalPrice"));
}
@@ -477,23 +479,41 @@
return goods;
}
-
- public static String getPDDGoodsId(String url) {
+
+ public static String getJDGoodsJS() {
+ InputStream input = JDUtil.class.getClassLoader().getResourceAsStream("pddGoods.js");
+ StringBuilder sb = new StringBuilder();
+ String line;
+ BufferedReader br = new BufferedReader(new InputStreamReader(input));
try {
- if (url.startsWith("https://mobile.yangkeduo.com/goods.html?")
- || url.startsWith("http://mobile.yangkeduo.com/goods.html?")||url.contains("yangkeduo.com/duo_coupon_landing.html?")) {
- String preUrl = url.substring(url.indexOf("?") + 1, url.length());
- String[] sts = preUrl.split("&");
- for (String st : sts) {
- if (st.startsWith("goods_id=")) {
- return st.split("=")[1].trim();
- }
- }
+ while ((line = br.readLine()) != null) {
+ sb.append(line);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return sb.toString();
+ }
+
+ static String jdGoodsJs = null;
+
+ public static String getPDDGoodsId(String url) {
+ if (jdGoodsJs == null)
+ jdGoodsJs = getJDGoodsJS();
+ ScriptEngineManager manager = new ScriptEngineManager();
+ ScriptEngine engine = manager.getEngineByName("javascript");
+ try {
+ engine.eval(jdGoodsJs);
+ if (engine instanceof Invocable) {
+ Invocable in = (Invocable) engine;
+ String goodsId = in.invokeFunction("getGoodsId", url).toString();
+ if (!StringUtil.isNullOrEmpty(goodsId))
+ return goodsId.trim();
}
} catch (Exception e) {
+ e.printStackTrace();
}
return null;
}
-
-
+
}
--
Gitblit v1.8.0