package com.yeshi.buwan.util.log; import com.google.gson.Gson; import com.qcloud.cos.model.COSObjectSummary; import com.yeshi.buwan.util.COSManager; import com.yeshi.buwan.util.HttpUtil; import com.yeshi.buwan.util.StringUtil; import com.yeshi.buwan.util.TimeUtil; import net.sf.json.JSONObject; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.zip.GZIPInputStream; /** * @author hxh * @title: SearchKeyLogUtil * @description: 搜索关键字日志帮助类 * @date 2022/11/10 17:28 */ public class SearchKeyLogUtil { private final static String BUCKET_NAME = "log-1255749512"; private static List download(String url) throws IOException { InputStream inputStream = HttpUtil.getAsInputStream(url); GZIPInputStream gzin = new GZIPInputStream(inputStream); ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); int num; byte[] buf = new byte[1024]; while ((num = gzin.read(buf, 0, buf.length)) != -1) { byteOutputStream.write(buf, 0, num); } List searchKeyList = new ArrayList<>(); String finalString = new String(byteOutputStream.toByteArray(),"UTF-8"); Scanner scanner = new Scanner(finalString); while (scanner.hasNextLine()) { String data = scanner.nextLine(); System.out.println(new String(data.getBytes("ISO-8859-1"), "UTF-8")); System.out.println(new String(data.getBytes("GBK"), "UTF-8")); System.out.println(new String(data.getBytes("GB2312"), "UTF-8")); JSONObject json = JSONObject.fromObject(data); if (!StringUtil.isNullOrEmpty(json.optString("message"))) { try { searchKeyList.add(new Gson().fromJson(json.optString("message"), SearchKey.class)); } catch (Exception e) { e.printStackTrace(); } } } scanner.close(); return searchKeyList; } public static List downLoadData(Date dateTime) throws IOException { String day = TimeUtil.getGernalTime(dateTime.getTime(), "yyyy/MM/dd"); List resultList = COSManager.getInstance().listObject(BUCKET_NAME, "ap-guangzhou-buwan-search-key-1652433911/" + day); List urlList = new ArrayList<>(); for (COSObjectSummary object : resultList) { String key = object.getKey(); String url = String.format("https://%s.cos.ap-guangzhou.myqcloud.com/%s", BUCKET_NAME, key); urlList.add(url); } //下载文件 List finalList = new ArrayList<>(); for (String url : urlList) { List list = download(url); finalList.addAll(list); } return finalList; } public static List rank(List keyList, int num) { Map countMap = new HashMap<>(); Set identitySet = new HashSet<>(); for (SearchKey key : keyList) { String id = key.getKey() + "-" + key.getDevice(); if (!identitySet.contains(id)) { identitySet.add(id); if (countMap.get(key.getKey()) == null) { countMap.put(key.getKey(), 0); } countMap.put(key.getKey(), countMap.get(key.getKey()) + 1); } } List rankList = new ArrayList<>(); for (Iterator its = countMap.keySet().iterator(); its.hasNext(); ) { String key = its.next(); rankList.add(new RankInfo(key, countMap.get(key))); } Comparator cm = ((o1, o2) -> o2.getCount() - o1.getCount()); Collections.sort(rankList, cm); if (rankList.size() > num) { return rankList.subList(0, num); } return rankList; } /** * @return java.util.List * @author hxh * @description 获取今日搜索排行 * @date 18:54 2022/11/10 * @param: num **/ public static List getTodayRankList(int num) throws IOException { List dataList = downLoadData(new Date()); List rankList = rank(dataList, 100); return rankList; } /** * @return java.util.List * @author hxh * @description 获取日期搜索排行 * @date 18:54 2022/11/10 * @param: num **/ public static List getDayRankList(Date date, int num) throws IOException { List dataList = downLoadData(date); List rankList = rank(dataList, 100); return rankList; } public static void main(String[] args) throws IOException { List rankList = getTodayRankList(100); for (RankInfo rank : rankList) { System.out.println(rank.getKey() + " : " + rank.getCount()); } } static class SearchKey { /** * key : 排球少年 * type : 153 * detailSystemId : 44 * device : 75f425d9-95e7-383a-914b-a33bad909280 * createTime : 1668010401229 */ private String key; private String type; private String detailSystemId; private String device; private long createTime; public String getKey() { return key; } public void setKey(String key) { this.key = key; } public String getType() { return type; } public void setType(String type) { this.type = type; } public String getDetailSystemId() { return detailSystemId; } public void setDetailSystemId(String detailSystemId) { this.detailSystemId = detailSystemId; } public String getDevice() { return device; } public void setDevice(String device) { this.device = device; } public long getCreateTime() { return createTime; } public void setCreateTime(long createTime) { this.createTime = createTime; } } public static class RankInfo { private String key; private int count; public RankInfo(String key, int count) { this.key = key; this.count = count; } public String getKey() { return key; } public void setKey(String key) { this.key = key; } public int getCount() { return count; } public void setCount(int count) { this.count = count; } } }