fanli/src/main/java/com/yeshi/fanli/controller/TestController.java
@@ -6,7 +6,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -19,6 +21,9 @@ import javax.script.ScriptEngineManager; import javax.servlet.http.HttpServletRequest; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.methods.GetMethod; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; @@ -488,4 +493,37 @@ } catch (Exception e) { } } @RequestMapping("getTaoBaoGoodsDetail") public void getTaoBaoGoodsDetail(Long id, String headers, PrintWriter out) { JSONObject data = new JSONObject(); data.put("itemNumId", id + ""); String url = ""; try { url = String.format("https://acs.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=%s", URLEncoder.encode(data.toString(), "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } HttpClient client = new HttpClient(); GetMethod gm = new GetMethod(url); String result = null; try { JSONObject headerJson = JSONObject.fromObject(headers); for (Iterator<String> its = headerJson.keys(); its.hasNext();) { String key = its.next(); gm.setRequestHeader(key, headerJson.optString(key)); } client.executeMethod(gm); result = gm.getResponseBodyAsString(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } out.print(result); } } fanli/src/main/java/com/yeshi/fanli/controller/web/TLJFreeBuyGoodsController.java
@@ -113,7 +113,7 @@ } // 最多10个商品 List<TLJFreeBuyGoods> listGoods = tljFreeBuyGoodsService.listByDay(day, 10); List<TLJFreeBuyGoods> listGoods = tljFreeBuyGoodsService.listByDay(day, 100); JSONArray array = new JSONArray(); BigDecimal fanLiRate = hongBaoManageService.getFanLiRate(); fanli/src/main/java/com/yeshi/fanli/dao/goods/jd/NYouHuiGoodsDao.java
New file @@ -0,0 +1,29 @@ package com.yeshi.fanli.dao.goods.jd; import java.util.ArrayList; import java.util.List; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Direction; import org.springframework.data.domain.Sort.Order; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Repository; import com.yeshi.fanli.dao.MongodbBaseDao; import com.yeshi.fanli.entity.goods.jd.NYouHuiGoods; @Repository public class NYouHuiGoodsDao extends MongodbBaseDao<NYouHuiGoods> { public NYouHuiGoods selectByName(String name) { Query query = new Query(); query.addCriteria(Criteria.where("name").is(name)); List<Order> orders = new ArrayList<>(); orders.add(new Order(Direction.DESC, "updateTime")); query.with(new Sort(orders)); query.limit(1); return mongoTemplate.findOne(query, NYouHuiGoods.class); } } fanli/src/main/java/com/yeshi/fanli/entity/goods/jd/NYouHuiGoods.java
File was renamed from fanli/src/main/java/com/yeshi/fanli/dao/goods/jd/NYouHuiGoods.java @@ -1,4 +1,4 @@ package com.yeshi.fanli.dao.goods.jd; package com.yeshi.fanli.entity.goods.jd; import java.util.Date; import java.util.List; @@ -25,6 +25,16 @@ private Date updateTime; @Field private String className; @Field private List<Long> skuList;// 商品列表 public List<Long> getSkuList() { return skuList; } public void setSkuList(List<Long> skuList) { this.skuList = skuList; } public String getClassName() { return className; fanli/src/main/java/com/yeshi/fanli/job/JDGoodsUpdateJob.java
New file @@ -0,0 +1,78 @@ package com.yeshi.fanli.job; import java.util.ArrayList; import java.util.List; import javax.annotation.Resource; import org.springframework.stereotype.Component; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import com.yeshi.fanli.entity.goods.jd.NYouHuiGoods; import com.yeshi.fanli.log.LogHelper; import com.yeshi.fanli.service.inter.goods.jd.NYouHuiService; import com.yeshi.fanli.util.StringUtil; import com.yeshi.fanli.util.TimeUtil; import com.yeshi.fanli.util.goods.jd.NYouHuiUtil; import com.yeshi.fanli.util.jd.JDUtil; /** * 京东商品更新 * * @author Administrator * */ @Component public class JDGoodsUpdateJob { @Resource private NYouHuiService nyouHuiService; /** * 更新内优惠最近几天的商品 * * @param param * @return * @throws Exception */ @XxlJob("nyhLatest2DayGoodsUpdateHandler") public ReturnT<String> updateNYHLatest2DayGoods(String param) throws Exception { long startTime = TimeUtil.convertToTimeTemp( TimeUtil.getGernalTime(System.currentTimeMillis() - 1000 * 60 * 60 * 24L, "yyyy-MM-dd"), "yyyy-MM-dd"); List<String> list = NYouHuiUtil.getClasses(); for (String name : list) { try { List<NYouHuiGoods> goodsList = NYouHuiUtil.listByClassName(name, StringUtil.isNullOrEmpty(param) ? 1 : Integer.parseInt(param)); for (NYouHuiGoods goods : goodsList) { try { if (TimeUtil.convertToTimeTemp(goods.getPublishTime(), "yyyy-MM-dd") >= startTime) { NYouHuiGoods old = nyouHuiService.selectByName(goods.getName()); if (old == null || TimeUtil.convertToTimeTemp(old.getPublishTime(), "yyyy-MM-dd") < startTime) { NYouHuiGoods goodsDetail = NYouHuiUtil.getGoodsDetail(goods.getSourceUrl()); List<Long> skuList = new ArrayList<>(); for (String link : goodsDetail.getLinkList()) { String skuId = JDUtil.getJDGoodsIdByUJD(link); skuList.add(Long.parseLong(skuId)); } goodsDetail.setSkuId(skuList.get(0)); goodsDetail.setSkuList(skuList); goodsDetail.setClassName(name); nyouHuiService.addGoods(goodsDetail); Thread.sleep(2000 + (int) (Math.random() * 5) * 1000); } } } catch (Exception e) { LogHelper.errorDetailInfo(e, "京东-内优惠出错", goods.getSourceUrl()); } } } catch (Exception e) { LogHelper.errorDetailInfo(e, "京东-内优惠出错", name); } } return ReturnT.SUCCESS; } } fanli/src/main/java/com/yeshi/fanli/service/impl/goods/jd/NYouHuiServiceImpl.java
New file @@ -0,0 +1,37 @@ package com.yeshi.fanli.service.impl.goods.jd; import java.util.Date; import javax.annotation.Resource; import org.springframework.stereotype.Service; import com.yeshi.fanli.dao.goods.jd.NYouHuiGoodsDao; import com.yeshi.fanli.entity.goods.jd.NYouHuiGoods; import com.yeshi.fanli.service.inter.goods.jd.NYouHuiService; import com.yeshi.fanli.util.StringUtil; @Service public class NYouHuiServiceImpl implements NYouHuiService { @Resource private NYouHuiGoodsDao nyouHuiGoodsDao; @Override public void addGoods(NYouHuiGoods goods) { if (goods == null) return; if (goods.getSkuId() == null || StringUtil.isNullOrEmpty(goods.getDesc()) || StringUtil.isNullOrEmpty(goods.getName())) return; if(goods.getUpdateTime()==null) goods.setUpdateTime(new Date()); nyouHuiGoodsDao.save(goods); } @Override public NYouHuiGoods selectByName(String name) { return nyouHuiGoodsDao.selectByName(name); } } fanli/src/main/java/com/yeshi/fanli/service/inter/goods/jd/NYouHuiService.java
New file @@ -0,0 +1,22 @@ package com.yeshi.fanli.service.inter.goods.jd; import com.yeshi.fanli.entity.goods.jd.NYouHuiGoods; public interface NYouHuiService { /** * 添加商品 * * @param goods */ public void addGoods(NYouHuiGoods goods); /** * 根据名称查询 * * @param name * @return */ public NYouHuiGoods selectByName(String name); } fanli/src/main/java/com/yeshi/fanli/util/account/UserUtil.java
@@ -77,7 +77,7 @@ public static String getInviteCode(int base, Long uid) { long num = base * 1000000000L + uid ; long num = base * 100000000L + uid ; return convert10To36(num); } fanli/src/main/java/com/yeshi/fanli/util/goods/jd/NYouHuiUtil.java
@@ -13,7 +13,7 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.yeshi.fanli.dao.goods.jd.NYouHuiGoods; import com.yeshi.fanli.entity.goods.jd.NYouHuiGoods; /** * 京东内优惠商品爬取 https://www.n-youhui.com @@ -43,7 +43,7 @@ Elements level0 = classesRoot.getElementsByClass("level-0"); for (int i = 0; i < level0.size(); i++) { String name = level0.get(i).html().replace(" ", "").replaceAll("\\(([0-9]|,)*\\)", "").trim(); classList.add(name); classList.add(name.replace("/", "-")); } return classList; fanli/src/main/java/com/yeshi/fanli/util/taobao/TaoBaoUtil.java
@@ -1,6 +1,5 @@ package com.yeshi.fanli.util.taobao; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.reflect.Type; import java.math.BigDecimal; @@ -23,7 +22,6 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; @@ -1856,31 +1854,28 @@ String url = ""; try { url = String.format("https://acs.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=%s", url = String.format( "http://acs.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=%s&qq-pf-to=pcqq.group", URLEncoder.encode(data.toString(), "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } HttpClient client = new HttpClient(); GetMethod gm = new GetMethod(url); String result = null; try { gm.setRequestHeader("user-agent", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Mobile Safari/537.36"); gm.setRequestHeader("cache-control", "max-age=0"); gm.setRequestHeader("sec-fetch-mode", "navigate"); gm.setRequestHeader("sec-fetch-site", "none"); gm.setRequestHeader("sec-fetch-user", "?1"); gm.setRequestHeader("upgrade-insecure-requests", "1"); gm.setRequestHeader("cookie", "t=2ef08e4a12314c12dcd98d2ec480dd69; thw=cn; hng=CN%7Czh-CN%7CCNY%7C156; x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0; ali_ab=113.251.19.3.1554796147005.0; UM_distinctid=16b9d8257dd388-0a96f36f947888-e343166-1fa400-16b9d8257df114; WAPFDFDTGFG=%2B4cMKKP%2B8PI%2BuhLvELc2mB6zAz5Z54E%3D; _w_app_lg=0; enc=00wU0%2BKYu9Yf1tHSawG%2Bc4aMZojqg8eB6baoMr9Z8Ir4qJ6hRhMhdWQxbM71vvxv6OCsxN50T0xA1gJGIdFm7w%3D%3D; cna=Q9eYFEM1rm8CAXH7EKUDSamu; miid=769739881160810491; lgc=%5Cu6930%5Cu89C6%5Cu79D1%5Cu6280; tracknick=%5Cu6930%5Cu89C6%5Cu79D1%5Cu6280; tg=0; mt=ci=7_1; cookie2=13ad477d84d5e6659aa7855de3617163; _tb_token_=ee895586817b0; dnk=%5Cu6930%5Cu89C6%5Cu79D1%5Cu6280; v=0; _m_h5_tk=286f649add5207dfe82e770832daf1cc_1573547306314; _m_h5_tk_enc=137c39dafc279e40edc796770d3eda8f; tk_trace=oTRxOWSBNwn9dPyorMJE%2FoPdY8zZPEr%2FCrvCMS%2BG3sTRRWrQ%2BVVTl09ME1KrXdbd3QNiuPM1YZeXJrHpvLuG%2FSvA2umbWq5jqrcw%2FDpk1Lj6yQFVptx6jw71DzuBbw%2BuZLEozdKGb%2BZr85uzSnUSWfVAqZprzoB%2Bfde2IHnHJp651xghkKpmsSBgY5muhOI7FvNfc8zH1BIvygVQSEIa1bCCgy2VJl%2FkyuTXaGHb%2FAKxL566%2FLult1lRlo8RZx9s76BhXe7PTot5MPGrayPkRiqWfVdg3C7gFWRjrjjKriN47mbQazOoIM7knBX%2B3mnnmwWbCQjKiuA6I8W9ng%3D%3D; _cc_=VFC%2FuZ9ajQ%3D%3D; unb=3327215652; uc1=lng=zh_CN&tag=8&cookie16=W5iHLLyFPlMGbLDwA%2BdvAGZqLg%3D%3D&cookie15=VT5L2FSpMGV7TQ%3D%3D&cookie14=UoTbnrCeRJd%2F2g%3D%3D&pas=0&existShop=false&cookie21=VFC%2FuZ9aj38X; uc3=vt3=F8dByuWjrX8oBIwXl2Q%3D&nk2=sbdfkkkB37A%3D&id2=UNN%2F6whEupDGWQ%3D%3D&lg2=WqG3DMC9VAQiUQ%3D%3D; csg=30373d82; cookie17=UNN%2F6whEupDGWQ%3D%3D; skt=6a0617b2d0843ef4; existShop=MTU3MzgxNDAxNw%3D%3D; uc4=id4=0%40UgQ3BPth2rlXv3PxjwesD5MgnRD3&nk4=0%40s9JQzQv3gdCkG6YVO6ulkEbNug%3D%3D; _l_g_=Ug%3D%3D; sg=%E6%8A%8021; _nk_=%5Cu6930%5Cu89C6%5Cu79D1%5Cu6280; cookie1=AnRaCZfOZrWoY8u%2BoXxB8y%2FnnwnZszWqI6Bbr5BrFtM%3D; isg=BOnpxPqJkNY3p6zvIy1AdcEC-JVDtt3of1m6aoveZVAPUglk0wbtuNdAFL5BUXUg; l=dBgUvWMcqlf1dR9EBOCanurza77OSIRYYuPzaNbMi_5IT6T6q1QOkBPUkF96VfWfTVLB47_ypV99-etkZ2ejm8IpXUJsAMc."); client.executeMethod(gm); result = gm.getResponseBodyAsString(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); Connection.Response doc = Jsoup.connect(url).ignoreContentType(true) .timeout(1000*10) .userAgent( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18362") .header("Accept", "text/html, application/xhtml+xml, application/xml; q=0.9, */*; q=0.8") .header("Accept-Language", "zh-CN").header("Host", "acs.m.taobao.com") .header("Upgrade-Insecure-Requests", "1") .header("Cookie", "_cc_=VFC%2FuZ9ajQ%3D%3D; isg=BLu7TvdFYk89dV4DMXO7XNRGU58lEM8SDjV-ja14l7rRDNvuNeBfYtlOJmznNycK; thw=cn; miid=1490566681358073134; x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0; cna=ZW8rFCpNP1gCAWpXAiKqMzqi; tracknick=tb23001560; l=cBxbffilq1UJdAFCBOCanurza77OSdAYYuPzaNbMi_5NE6T1BR7Ok6G1vF96VsWdOW8B4NSiTkp9-etkZ3Znq9SpXUJ1.; tg=0; _w_app_lg=19; WAPFDFDTGFG=%2B4cMKKP%2B8PI%2BtNYpkiAuTPLkiJB1kcwi") .execute(); result = doc.body(); } catch (Exception e1) { e1.printStackTrace(); } JSONObject resultOBJ = JSONObject.fromObject(result);
