精选库更新

yujian

2019-05-28 4730ad35bb6b01ebf197bff15ec518182e29e059

精选库更新

 fanli/src/main/java/com/yeshi/fanli/job/QualityFactoryJob.java

@@ -8,8 +8,6 @@


import javax.annotation.Resource;



import net.sf.json.JSONObject;



import org.quartz.Job;

import org.quartz.JobDataMap;

import org.quartz.JobExecutionContext;

@@ -30,6 +28,8 @@
import com.yeshi.fanli.util.StringUtil;

import com.yeshi.fanli.util.taobao.TaoBaoUtil;

import com.yeshi.fanli.util.taobao.TaoKeApiUtil;



import net.sf.json.JSONObject;



/**

 * 精选商品自动入库

@@ -59,8 +59,6 @@


    @Override

    public void execute(JobExecutionContext context) throws JobExecutionException {

        LogHelper.test("---UpdateQualityFactoryJob----execute ");



        // 爬取商品信息

        searchGoods(context);

    }

@@ -90,7 +88,8 @@
            }



            /* 来源名称 */

            if (jsonData.get("sourceCalss") != null && !StringUtil.isNullOrEmpty(jsonData.get("sourceCalss").toString())) {

            if (jsonData.get("sourceCalss") != null

                    && !StringUtil.isNullOrEmpty(jsonData.get("sourceCalss").toString())) {

                String sourceCalss = jsonData.get("sourceCalss").toString();

                autoRule.setSourceCalss(sourceCalss);

            }

@@ -116,41 +115,34 @@
            /* 标签列表 */

            List<Label> listLabs = new ArrayList<Label>();



            if (jsonData.get("lableNames") != null && !StringUtil.isNullOrEmpty(jsonData.get("lableNames").toString())) {

            if (jsonData.get("lableNames") != null

                    && !StringUtil.isNullOrEmpty(jsonData.get("lableNames").toString())) {



                // 空格隔开

                String[] arrtitles = jsonData.get("lableNames").toString().split("\\s+");



                /* 遍历标签名称：查询数据库中是否已存在 */

                // 获取标签属性

                for (String name : arrtitles) {

                    List<Label> labels = labelService.selectByTitle(name.trim());



                    if (labels != null && labels.size() > 0) {

                        Label label = labels.get(0);



                        listLabs.add(label);

                    } else {

                        // 新增

                        Label addLabel = addLabel(name, admin);

                        listLabs.add(addLabel);

                    Label label = labelService.getByTitleCache("labKey", name.trim());

                    if (label == null) {

                        label = insertLabel(name, admin);

                    }



                    listLabs.add(label);

                }

            }



            int source = Integer.parseInt(jobDataMap.get("source").toString());



            autoRule.setSource(QualityFactory.SOURCE_TAOBAO);



            // 1、来源淘宝普通接口

            if (BoutiqueAutoRule.TB_OPTIONAL == source) {

                // 来源淘宝普通接口

                autoRule.setGoodsSource(QualityFactory.SOURCE_TAOBAO);



                searchTaoBao(jsonData, listLabs, autoRule);

            }



            // 2、淘宝推荐

            if (BoutiqueAutoRule.TB_MATERIAL == source) {

                autoRule.setGoodsSource(QualityFactory.SOURCE_TAOBAO_MATERIAL);

                // 淘宝推荐

                searchMaterial(jsonData, listLabs, autoRule);

            }



@@ -171,7 +163,6 @@
     * @param jobDataMap

     */

    private void searchTaoBao(JSONObject jsonData, List<Label> listLabs, BoutiqueAutoRule autoRule) {



        try {

            Date time = new Date();

            String formatDate = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(time);

@@ -205,16 +196,13 @@
            filter.setPageSize(pageSize);



            for (int page = 1; page <= pageMax; page++) {



                filter.setPage(page);



                TaoBaoSearchResult result = TaoKeApiUtil.searchWuLiao(filter);

                LogHelper.test(title + "searchWuLiao 已爬取- " + page + "-页-");



                // 结果为空 时不再请求

                // 结果为空 结束请求

                if (result == null) {

                    LogHelper.test(title + "searchWuLiao返回 result为空");

                    return;

                    break;

                }



                // 结果数据处理

@@ -226,8 +214,8 @@


                /* 权重范围 */

                if (page < 3) {

                    startWeight = 10000;

                    endWeight = 12000;

                    startWeight = 4000;

                    endWeight = 6000;

                } else {

                    startWeight = 10;

                    endWeight = 3000;

@@ -237,14 +225,10 @@
                autoRule.setEndWeight(endWeight);



                /* 筛选商品进入精选库中 */

                if (StringUtil.isNullOrEmpty(title)) {

                    screen(listGoods, jsonData, filter, listLabs, autoRule);

                if (!StringUtil.isNullOrEmpty(title) && title.contains(IS_9K9)) {

                    screen9K9(filter, listGoods, jsonData, listLabs, autoRule);

                } else {

                    if (title.contains(IS_9K9)) {

                        screen9K9(filter, listGoods, jsonData, listLabs, autoRule);

                    } else {

                        screen(listGoods, jsonData, filter, listLabs, autoRule);

                    }

                    screen(listGoods, jsonData, filter, listLabs, autoRule);

                }



                /* 销售量小值 */

@@ -287,50 +271,16 @@
                    break;// 结束爬取数据

                }

            }

			
            // 9k9 限时抢购  不操作删除

            if (title.contains(IS_9K9) || title.contains(FLASH_SALE)) {



            // 9k9 限时抢购 不操作删除

            if (!StringUtil.isNullOrEmpty(title) && (title.contains(IS_9K9) || title.contains(FLASH_SALE))) {

                return;

            }

			


            /* 更新完成后清理之前创建的数据 */

            if (jsonData.get("systemCid") != null && !StringUtil.isNullOrEmpty(jsonData.get("systemCid").toString())) {

				
                Long systemCid = Long.parseLong(jsonData.get("systemCid").toString());

			
                List<Long> list = qualityFactoryService.getAuctionIdbyClassId(systemCid, autoRule.getGoodsSource(), formatDate);

				
                if (list != null && list.size() > 0) {

                    if (list.size() <= 100) {

                        qualityFactoryService.deleteBatchByTaoBaoGoodsId(list);

                    } else {



                        int start = 0;

                        int end = 100;

                        int countRow = 100;



                        int count = list.size() / countRow;



                        if (list.size() % countRow > 0) {

                            count += 1;

                        }



                        for (int i = 0; i < count; i++) {

                            List<Long> sublist = new ArrayList<Long>();

                            if (i == count - 1) {

                                //System.out.println(list.subList(start, list.size()));

                                sublist = list.subList(start, list.size());

                                qualityFactoryService.deleteBatchByTaoBaoGoodsId(sublist);

                                break;

                            }

                            sublist = list.subList(start, end);

                            qualityFactoryService.deleteBatchByTaoBaoGoodsId(sublist);

                            start += countRow;

                            end += countRow;

                        }



                    }

                }

                deleteNotUpdateGoods(systemCid, formatDate, autoRule.getGoodsSource());

            }



        } catch (Exception e) {

@@ -405,14 +355,10 @@
                }



                /* 筛选商品进入精选库中 */

                if (StringUtil.isNullOrEmpty(title)) {

                    screen(listGoods, jsonData, filter, listLabs, autoRule);

                if (!StringUtil.isNullOrEmpty(title) && title.contains(IS_9K9)) {

                    screen9K9(filter, listGoods, jsonData, listLabs, autoRule);

                } else {

                    if (title.contains(IS_9K9)) {

                        screen9K9(filter, listGoods, jsonData, listLabs, autoRule);

                    } else {

                        screen(listGoods, jsonData, filter, listLabs, autoRule);

                    }

                    screen(listGoods, jsonData, filter, listLabs, autoRule);

                }



                // 总页数爬取完成

@@ -447,7 +393,6 @@
     * @return

     */

    public SearchFilter getSearchFilter(JSONObject jsonData) {



        SearchFilter filter = new SearchFilter();



        // 排序字段 销量高到低(默认)

@@ -536,8 +481,154 @@
            // 退款率是否低于行业均值 1 有 0 无

            filter.setIncludeRfdRate(true);

        }



        return filter;

    }



    /**

     * 9.9 19.9 29.9 39.9 商品筛选

     * 
     * @param taoBaoGoodsBriefs

     */

    public void screen9K9(SearchFilter filter, List<TaoBaoGoodsBrief> taoBaoGoodsBriefs, JSONObject jsonData,

            List<Label> listLabs, BoutiqueAutoRule autoRule) throws Exception {



        /* 9k9 */

        BigDecimal price9k9 = new BigDecimal(10);

        List<TaoBaoGoodsBrief> list9k9 = new ArrayList<TaoBaoGoodsBrief>();



        /* 19k9 */

        BigDecimal price19k9 = new BigDecimal(20);

        List<TaoBaoGoodsBrief> list19k9 = new ArrayList<TaoBaoGoodsBrief>();



        /* 29.9 */

        BigDecimal price29K9 = new BigDecimal(30);

        List<TaoBaoGoodsBrief> list29k9 = new ArrayList<TaoBaoGoodsBrief>();



        /* 49.9 */

        BigDecimal price49K9 = new BigDecimal(50);

        List<TaoBaoGoodsBrief> list49k9 = new ArrayList<TaoBaoGoodsBrief>();



        /* 销售量小值 */

        int minSales = 0;

        if (jsonData.get("startSales") != null) {

            minSales = (int) jsonData.get("startSales");

        }



        /* 销售量大值 */

        int maxSales = 0;

        if (jsonData.get("endSales") != null) {

            maxSales = (int) jsonData.get("endSales");

        }



        // 验证是否符合要求入库

        for (TaoBaoGoodsBrief goodsBrief : taoBaoGoodsBriefs) {



            /* 验证在是否有券 */

            if (filter.getQuan() > 0) {

                String couponInfo = goodsBrief.getCouponInfo();

                if (StringUtil.isNullOrEmpty(couponInfo)) {

                    continue;

                }

            }



            /* 验证销售量 */

            int biz30day = goodsBrief.getBiz30day();

            if (maxSales > 0 && minSales > 0 && (biz30day < minSales || biz30day > maxSales)) {

                continue;

            } else if (maxSales > 0 && biz30day > maxSales) {

                continue;

            } else if (minSales > 0 && biz30day < minSales) {

                continue;

            }



            /* 验证比例 */

            if (BoutiqueAutoRule.TB_MATERIAL == autoRule.getSource()) {

                int startTkRate = filter.getStartTkRate();

                int endTkRate = filter.getEndTkRate();

                BigDecimal tkRate = goodsBrief.getTkRate();

                if (startTkRate > 0 && endTkRate > 0 && (tkRate.compareTo(new BigDecimal(startTkRate / 100)) < 0

                        || tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0)) {

                    continue;

                } else if (endTkRate > 0 && tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0) {

                    continue;

                } else if (startTkRate > 0 && tkRate.compareTo(new BigDecimal(startTkRate / 100)) < 0) {

                    continue;

                }

            }



            /* 券后价--计算 */

            BigDecimal couponPrice = TaoBaoUtil.getAfterUseCouplePrice(goodsBrief);

            if (couponPrice.compareTo(price9k9) < 0) {

                list9k9.add(goodsBrief);

            } else if (couponPrice.compareTo(price19k9) < 0) {

                list19k9.add(goodsBrief);

            } else if (couponPrice.compareTo(price29K9) < 0) {

                list29k9.add(goodsBrief);

            } else if (couponPrice.compareTo(price49K9) < 0) {

                list49k9.add(goodsBrief);

            }

        }

        autoRule.setCalss9k9(true);

        AdminUser admin = autoRule.getAdminUser();



        if (list9k9.size() > 0) {

            Label label = labelService.getByTitleCache("labKey", "9.9");

            if (label == null) {

                label = insertLabel("9.9", admin);

            }

            qualityFactoryService.autoInsertOrUpadateStorage(autoRule, list9k9, listLabs, label);

        }



        if (list19k9.size() > 0) {

            Label label = labelService.getByTitleCache("labKey", "19.9");

            if (label == null) {

                label = insertLabel("9.9", admin);

            }

            qualityFactoryService.autoInsertOrUpadateStorage(autoRule, list19k9, listLabs, label);

        }



        if (list29k9.size() > 0) {

            Label label = labelService.getByTitleCache("labKey", "29.9");

            if (label == null) {

                label = insertLabel("9.9", admin);

            }

            qualityFactoryService.autoInsertOrUpadateStorage(autoRule, list29k9, listLabs, label);

        }



        if (list49k9.size() > 0) {

            Label label = labelService.getByTitleCache("labKey", "49.9");

            if (label == null) {

                label = insertLabel("9.9", admin);

            }

            qualityFactoryService.autoInsertOrUpadateStorage(autoRule, list49k9, listLabs, label);

        }

    }



    /**

     * 筛选符合条件的商品

     * 
     * @param taoBaoGoodsBriefs

     * @param jsonData

     * @param filter

     * @param listLabs

     * @param autoRule

     * @throws Exception

     */

    public void screen(List<TaoBaoGoodsBrief> taoBaoGoodsBriefs, JSONObject jsonData, SearchFilter filter,

            List<Label> listLabs, BoutiqueAutoRule autoRule) throws Exception {



        List<TaoBaoGoodsBrief> listGoods = new ArrayList<TaoBaoGoodsBrief>();

        for (TaoBaoGoodsBrief goodsBrief : taoBaoGoodsBriefs) {

            // 验证是否符合要求

            if (validate(filter, jsonData, goodsBrief, 1)) {

                listGoods.add(goodsBrief);

            }

        }



        if (listGoods.size() > 0) {

            autoRule.setCalss9k9(false);

            qualityFactoryService.autoInsertOrUpadateStorage(autoRule, listGoods, listLabs, null);

        }

    }



    /**

@@ -546,7 +637,6 @@
     * @return

     */

    public boolean validate(SearchFilter filter, JSONObject jsonData, TaoBaoGoodsBrief goodsBrief, int source) {



        /* 销售量小值 */

        int minSales = 0;

        if (jsonData.get("startSales") != null) {

@@ -578,8 +668,8 @@


            BigDecimal tkRate = goodsBrief.getTkRate();



            if (startTkRate > 0 && endTkRate > 0

                    && (tkRate.compareTo(new BigDecimal(startTkRate / 100)) < 0 || tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0)) {

            if (startTkRate > 0 && endTkRate > 0 && (tkRate.compareTo(new BigDecimal(startTkRate / 100)) < 0

                    || tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0)) {

                return false;

            } else if (endTkRate > 0 && tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0) {

                return false;

@@ -604,7 +694,8 @@
        BigDecimal couponPrice = TaoBaoUtil.getAfterUseCouplePrice(goodsBrief);



        /* 验证在券后价 */

        if (startPrice != null && endPrice != null && (couponPrice.compareTo(startPrice) < 0 || couponPrice.compareTo(endPrice) > 0)) {

        if (startPrice != null && endPrice != null

                && (couponPrice.compareTo(startPrice) < 0 || couponPrice.compareTo(endPrice) > 0)) {

            return false;

        } else if (endPrice != null && couponPrice.compareTo(endPrice) > 0) {

            return false;

@@ -624,188 +715,14 @@
    }



    /**

     * 9.9 19.9 29.9 39.9 商品筛选

     * 插入新的标签

     * 

     * @param taoBaoGoodsBriefs

     * @param title

     * @param admin

     * @return

     * @throws Exception

     */

    public void screen9K9(SearchFilter filter, List<TaoBaoGoodsBrief> taoBaoGoodsBriefs, JSONObject jsonData, List<Label> listLabs,

            BoutiqueAutoRule autoRule) throws Exception {



        // 9k9数据-

        List<TaoBaoGoodsBrief> list9k9 = new ArrayList<TaoBaoGoodsBrief>();



        // 19k9数据-

        List<TaoBaoGoodsBrief> list19k9 = new ArrayList<TaoBaoGoodsBrief>();



        // 29k9数据-

        List<TaoBaoGoodsBrief> list29k9 = new ArrayList<TaoBaoGoodsBrief>();



        // 49k9数据-

        List<TaoBaoGoodsBrief> list49k9 = new ArrayList<TaoBaoGoodsBrief>();



        // 验证是否符合要求入库

        for (TaoBaoGoodsBrief goodsBrief : taoBaoGoodsBriefs) {



            /* 销售量小值 */

            int minSales = 0;

            if (jsonData.get("startSales") != null) {

                minSales = (int) jsonData.get("startSales");

            }



            /* 销售量大值 */

            int maxSales = 0;

            if (jsonData.get("endSales") != null) {

                maxSales = (int) jsonData.get("endSales");

            }



            int biz30day = goodsBrief.getBiz30day();



            /* 验证销售量 */

            if (maxSales > 0 && minSales > 0 && (biz30day < minSales || biz30day > maxSales)) {

                continue;

            } else if (maxSales > 0 && biz30day > maxSales) {

                continue;

            } else if (minSales > 0 && biz30day < minSales) {

                continue;

            }



            /* 验证在是否有券 */

            if (filter.getQuan() > 0) {

                String couponInfo = goodsBrief.getCouponInfo();

                if (StringUtil.isNullOrEmpty(couponInfo)) {

                    continue;

                }

            }



            /* 验证比例 */

            if (BoutiqueAutoRule.TB_MATERIAL == autoRule.getSource()) {



                int startTkRate = filter.getStartTkRate();

                int endTkRate = filter.getEndTkRate();



                BigDecimal tkRate = goodsBrief.getTkRate();



                if (startTkRate > 0 && endTkRate > 0

                        && (tkRate.compareTo(new BigDecimal(startTkRate / 100)) < 0 || tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0)) {

                    continue;

                } else if (endTkRate > 0 && tkRate.compareTo(new BigDecimal(endTkRate / 100)) > 0) {

                    continue;

                } else if (startTkRate > 0 && tkRate.compareTo(new BigDecimal(startTkRate / 100)) < 0) {

                    continue;

                }

            }



            /* 9k9 */

            BigDecimal price9k9 = new BigDecimal(10);



            /* 19k9 */

            BigDecimal price19k9 = new BigDecimal(20);



            /* 29.9 */

            BigDecimal price29K9 = new BigDecimal(30);



            /* 49.9 */

            BigDecimal price49K9 = new BigDecimal(50);

            /* 券后价--计算 */

            BigDecimal couponPrice = TaoBaoUtil.getAfterUseCouplePrice(goodsBrief);



            if (couponPrice.compareTo(price9k9) < 0) {

                list9k9.add(goodsBrief);

            } else if (couponPrice.compareTo(price19k9) < 0) {

                list19k9.add(goodsBrief);

            } else if (couponPrice.compareTo(price29K9) < 0) {

                list29k9.add(goodsBrief);

            } else if (couponPrice.compareTo(price49K9) < 0) {

                list49k9.add(goodsBrief);

            }

        }



        autoRule.setCalss9k9(true);

        AdminUser admin = autoRule.getAdminUser();



        if (list9k9.size() > 0) {

            List<Label> labs9K9 = new ArrayList<Label>();

            labs9K9.addAll(listLabs);



            List<Label> list = labelService.selectByTitleCache("labKey", "9.9");



            if (list == null || list.size() == 0) {

                // 新增

                Label addLabel = addLabel("9.9", admin);

                labs9K9.add(addLabel);

            } else {

                labs9K9.add(list.get(0));

            }



            qualityFactoryService.autoInsertOrUpadateStorage(list9k9, labs9K9, autoRule);

        }



        if (list19k9.size() > 0) {

            List<Label> labs19K9 = new ArrayList<Label>();

            labs19K9.addAll(listLabs);

            List<Label> list = labelService.selectByTitleCache("labKey", "19.9");

            if (list == null || list.size() == 0) {

                // 新增

                Label addLabel = addLabel("19.9", admin);

                labs19K9.add(addLabel);

            } else {

                labs19K9.add(list.get(0));

            }

            qualityFactoryService.autoInsertOrUpadateStorage(list19k9, labs19K9, autoRule);

        }



        if (list29k9.size() > 0) {

            List<Label> labs29K9 = new ArrayList<Label>();

            labs29K9.addAll(listLabs);

            List<Label> list = labelService.selectByTitleCache("labKey", "29.9");

            if (list == null || list.size() == 0) {

                // 新增

                Label addLabel = addLabel("29.9", admin);

                labs29K9.add(addLabel);

            } else {

                labs29K9.add(list.get(0));

            }



            qualityFactoryService.autoInsertOrUpadateStorage(list29k9, labs29K9, autoRule);

        }



        if (list49k9.size() > 0) {

            List<Label> labs49K9 = new ArrayList<Label>();

            labs49K9.addAll(listLabs);

            List<Label> list = labelService.selectByTitleCache("labKey", "49.9");

            if (list == null || list.size() == 0) {

                // 新增

                Label addLabel = addLabel("49.9", admin);

                labs49K9.add(addLabel);

            } else {

                labs49K9.add(list.get(0));

            }



            qualityFactoryService.autoInsertOrUpadateStorage(list49k9, labs49K9, autoRule);

        }



    }



    public void screen(List<TaoBaoGoodsBrief> taoBaoGoodsBriefs, JSONObject jsonData, SearchFilter filter, List<Label> listLabs,

            BoutiqueAutoRule autoRule) throws Exception {



        // 遍历返回数据-

        List<TaoBaoGoodsBrief> listGoods = new ArrayList<TaoBaoGoodsBrief>();



        for (TaoBaoGoodsBrief goodsBrief : taoBaoGoodsBriefs) {

            // 验证是否符合要求

            if (validate(filter, jsonData, goodsBrief, 1)) {

                listGoods.add(goodsBrief);

            }

        }



        if (listGoods.size() > 0) {

            autoRule.setCalss9k9(false);

            qualityFactoryService.autoInsertOrUpadateStorage(listGoods, listLabs, autoRule);

        }

    }



    public Label addLabel(String title, AdminUser admin) throws Exception {

    public Label insertLabel(String title, AdminUser admin) throws Exception {

        // 新增

        Label label = new Label();

        label.setTitle(title);// 名称

@@ -816,8 +733,51 @@
        label.setIosClick(0l);

        label.setAndroidClick(0l);

        labelService.insertSelective(label);



        return label;

    }



    /**

     * 删除未更新的商品

     * 
     * @param systemCid

     * @param formatDate

     * @param source

     */

    public void deleteNotUpdateGoods(Long systemCid, String formatDate, int source) {

        List<Long> list = qualityFactoryService.getAuctionIdbyClassId(systemCid, source, formatDate);

        if (list == null || list.size() == 0) {

            return;

        }



        // 小于100条直接删除

        if (list.size() <= 100) {

            qualityFactoryService.deleteBatchByTaoBaoGoodsId(list);

            return;

        } 
		
		
        // 大于100需要分段删除

        int start = 0;

        int end = 100;

        int countRow = 100;



        int count = list.size() / countRow;

        if (list.size() % countRow > 0) {

            count += 1;

        }



        for (int i = 0; i < count; i++) {

            List<Long> sublist = new ArrayList<Long>();

            if (i == count - 1) {

                sublist = list.subList(start, list.size());

                qualityFactoryService.deleteBatchByTaoBaoGoodsId(sublist);

                break;

            }

            sublist = list.subList(start, end);

            qualityFactoryService.deleteBatchByTaoBaoGoodsId(sublist);

            start += countRow;

            end += countRow;

        }



    }

}


 fanli/src/main/java/com/yeshi/fanli/job/UpdateDaTaoKeJob.java

@@ -52,7 +52,7 @@
            autoRule.setStartWeight(1);

            autoRule.setEndWeight(3000);

            autoRule.setGoodsSource(QualityFactory.SOURCE_TAOBAO_DATAOKE);

            qualityFactoryService.autoInsertOrUpadateStorage(listgoods, null, autoRule);

            qualityFactoryService.autoInsertOrUpadateStorage(autoRule, listgoods, null, null);



        } catch (Exception e) {

            try {