package com.yeshi.buwan.util.log;
|
|
import com.google.gson.Gson;
|
import com.qcloud.cos.model.COSObjectSummary;
|
import com.yeshi.buwan.util.COSManager;
|
import com.yeshi.buwan.util.HttpUtil;
|
import com.yeshi.buwan.util.StringUtil;
|
import com.yeshi.buwan.util.TimeUtil;
|
import net.sf.json.JSONObject;
|
|
import java.io.ByteArrayOutputStream;
|
import java.io.IOException;
|
import java.io.InputStream;
|
import java.util.*;
|
import java.util.zip.GZIPInputStream;
|
|
/**
|
* @author hxh
|
* @title: SearchKeyLogUtil
|
* @description: 搜索关键字日志帮助类
|
* @date 2022/11/10 17:28
|
*/
|
public class SearchKeyLogUtil {
|
private final static String BUCKET_NAME = "log-1255749512";
|
|
private static List<SearchKey> download(String url) throws IOException {
|
InputStream inputStream = HttpUtil.getAsInputStream(url);
|
GZIPInputStream gzin = new GZIPInputStream(inputStream);
|
|
ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
|
int num;
|
byte[] buf = new byte[1024];
|
while ((num = gzin.read(buf, 0, buf.length)) != -1) {
|
byteOutputStream.write(buf, 0, num);
|
}
|
|
List<SearchKey> searchKeyList = new ArrayList<>();
|
String finalString
|
= new String(byteOutputStream.toByteArray(),"UTF-8");
|
Scanner scanner = new Scanner(finalString);
|
while (scanner.hasNextLine()) {
|
String data = scanner.nextLine();
|
System.out.println(new String(data.getBytes("ISO-8859-1"), "UTF-8"));
|
System.out.println(new String(data.getBytes("GBK"), "UTF-8"));
|
System.out.println(new String(data.getBytes("GB2312"), "UTF-8"));
|
JSONObject json = JSONObject.fromObject(data);
|
if (!StringUtil.isNullOrEmpty(json.optString("message"))) {
|
try {
|
searchKeyList.add(new Gson().fromJson(json.optString("message"), SearchKey.class));
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
scanner.close();
|
return searchKeyList;
|
}
|
|
public static List<SearchKey> downLoadData(Date dateTime) throws IOException {
|
String day = TimeUtil.getGernalTime(dateTime.getTime(), "yyyy/MM/dd");
|
List<COSObjectSummary> resultList = COSManager.getInstance().listObject(BUCKET_NAME, "ap-guangzhou-buwan-search-key-1652433911/" + day);
|
List<String> urlList = new ArrayList<>();
|
for (COSObjectSummary object : resultList) {
|
String key = object.getKey();
|
String url = String.format("https://%s.cos.ap-guangzhou.myqcloud.com/%s", BUCKET_NAME, key);
|
urlList.add(url);
|
}
|
//下载文件
|
List<SearchKey> finalList = new ArrayList<>();
|
for (String url : urlList) {
|
List<SearchKey> list = download(url);
|
finalList.addAll(list);
|
}
|
return finalList;
|
}
|
|
public static List<RankInfo> rank(List<SearchKey> keyList, int num) {
|
Map<String, Integer> countMap = new HashMap<>();
|
Set<String> identitySet = new HashSet<>();
|
for (SearchKey key : keyList) {
|
String id = key.getKey() + "-" + key.getDevice();
|
if (!identitySet.contains(id)) {
|
identitySet.add(id);
|
if (countMap.get(key.getKey()) == null) {
|
countMap.put(key.getKey(), 0);
|
}
|
countMap.put(key.getKey(), countMap.get(key.getKey()) + 1);
|
}
|
}
|
List<RankInfo> rankList = new ArrayList<>();
|
for (Iterator<String> its = countMap.keySet().iterator(); its.hasNext(); ) {
|
String key = its.next();
|
rankList.add(new RankInfo(key, countMap.get(key)));
|
}
|
Comparator<RankInfo> cm = ((o1, o2) -> o2.getCount() - o1.getCount());
|
Collections.sort(rankList, cm);
|
if (rankList.size() > num) {
|
return rankList.subList(0, num);
|
}
|
return rankList;
|
}
|
|
/**
|
* @return java.util.List<com.yeshi.buwan.util.log.SearchKeyLogUtil.RankInfo>
|
* @author hxh
|
* @description 获取今日搜索排行
|
* @date 18:54 2022/11/10
|
* @param: num
|
**/
|
public static List<RankInfo> getTodayRankList(int num) throws IOException {
|
List<SearchKey> dataList = downLoadData(new Date());
|
List<RankInfo> rankList = rank(dataList, 100);
|
return rankList;
|
}
|
|
/**
|
* @return java.util.List<com.yeshi.buwan.util.log.SearchKeyLogUtil.RankInfo>
|
* @author hxh
|
* @description 获取日期搜索排行
|
* @date 18:54 2022/11/10
|
* @param: num
|
**/
|
public static List<RankInfo> getDayRankList(Date date, int num) throws IOException {
|
List<SearchKey> dataList = downLoadData(date);
|
List<RankInfo> rankList = rank(dataList, 100);
|
return rankList;
|
}
|
|
|
public static void main(String[] args) throws IOException {
|
List<RankInfo> rankList = getTodayRankList(100);
|
for (RankInfo rank : rankList) {
|
System.out.println(rank.getKey() + " : " + rank.getCount());
|
}
|
}
|
|
|
static class SearchKey {
|
/**
|
* key : 排球少年
|
* type : 153
|
* detailSystemId : 44
|
* device : 75f425d9-95e7-383a-914b-a33bad909280
|
* createTime : 1668010401229
|
*/
|
|
private String key;
|
private String type;
|
private String detailSystemId;
|
private String device;
|
private long createTime;
|
|
public String getKey() {
|
return key;
|
}
|
|
public void setKey(String key) {
|
this.key = key;
|
}
|
|
public String getType() {
|
return type;
|
}
|
|
public void setType(String type) {
|
this.type = type;
|
}
|
|
public String getDetailSystemId() {
|
return detailSystemId;
|
}
|
|
public void setDetailSystemId(String detailSystemId) {
|
this.detailSystemId = detailSystemId;
|
}
|
|
public String getDevice() {
|
return device;
|
}
|
|
public void setDevice(String device) {
|
this.device = device;
|
}
|
|
public long getCreateTime() {
|
return createTime;
|
}
|
|
public void setCreateTime(long createTime) {
|
this.createTime = createTime;
|
}
|
}
|
|
public static class RankInfo {
|
private String key;
|
private int count;
|
|
public RankInfo(String key, int count) {
|
this.key = key;
|
this.count = count;
|
}
|
|
public String getKey() {
|
return key;
|
}
|
|
public void setKey(String key) {
|
this.key = key;
|
}
|
|
public int getCount() {
|
return count;
|
}
|
|
public void setCount(int count) {
|
this.count = count;
|
}
|
}
|
}
|
