package org.yeshi.utils; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; import java.security.KeyStore; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.multipart.FilePart; import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.commons.httpclient.methods.multipart.StringPart; import org.apache.commons.io.FileUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.yeshi.utils.entity.ProxyIP; import net.sf.json.JSONObject; public class HttpUtil { public static List getUrlListFromText(String text) { List urlList = new ArrayList(); // 链接 String URL_REGEX = "(((http|https)://)|(www\\.))[a-zA-Z0-9\\._-]+\\.[a-zA-Z]{2,6}(:[0-9]{1,4})?(/[a-zA-Z0-9\\&%_\\./-~-]*)?"; Pattern p = Pattern.compile(URL_REGEX); Matcher matcher = p.matcher(text); while (matcher.find()) { String url = matcher.group(); if (!StringUtil.isNullOrEmpty(url)) { urlList.add(url); } } return urlList; } /** * 获取链接中的参数 * * @param url * @return */ public static Map getPramsFromUrl(String url) { Map params = new HashMap<>(); if (url != null && url.indexOf("?") > -1) { String paramStr = url.split("\\?")[1]; String[] sts = paramStr.split("&"); for (String st : sts) { if (st.indexOf("=") > -1) { params.put(st.split("=")[0], st.split("=")[1]); } } } return params; } public static String getHost(String host) { try { URL url = new URL(host); return url.getHost(); } catch (MalformedURLException e) { e.printStackTrace(); } return ""; } public static String getShortLink(String url) { // return url; String shortLink = getShortLink3(url); if (StringUtil.isNullOrEmpty(shortLink)) shortLink = getShortLink1(url); if (StringUtil.isNullOrEmpty(shortLink)) shortLink = getShortLink2(url); return shortLink; } private static String getShortLink1(String url) { try { String totalUrl = String.format( "http://suo.im/api.htm?format=json&url=%s&key=5d7728078e676d45275f816b@681bb0446f1e6af4f8fc6ce3cad2a684", URLEncoder.encode(url)); String result = get(totalUrl, null); System.out.println(result); try { JSONObject json = JSONObject.fromObject(result); return json.optString("url"); } catch (Exception e) { } } catch (Exception e) { } return null; } // 搜狐网址 private static String getShortLink2(String url) { try { String totalUrl = String.format("https://sohu.gg/api/?key=2zumZxsL8MuX&url=%s", URLEncoder.encode(url, "UTF-8")); String result = get(totalUrl, null); if (result != null && result.startsWith("http")) return result; } catch (Exception e) { e.printStackTrace(); } return null; } private static String getShortLink3(String url) { // try { // String totalUrl = String.format("http://h5.flq001.com/short/createShort?url=%s", // URLEncoder.encode(url, "UTF-8")); // String result = get(totalUrl, null); // JSONObject data = JSONObject.fromObject(result); // return data.optString("short_url"); // } catch (Exception e) { // e.printStackTrace(); // } return null; } public static String get(String url) { return get(url, null); } /** * get请求 * * @param url * @param timeoutMS-请求超时时间 * @return */ public static String get(String url, int timeoutMS) { HttpClient client = new HttpClient(); client.getHttpConnectionManager().getParams().setConnectionTimeout(timeoutMS); try { GetMethod method = new GetMethod(url); client.executeMethod(method); return method.getResponseBodyAsString(); } catch (Exception e) { } return null; } public static String get(String url, Map headers) { HttpClient client = new HttpClient(); try { GetMethod method = new GetMethod(url); if (headers != null) { Iterator keys = headers.keySet().iterator(); while (keys.hasNext()) { String key = keys.next(); method.addRequestHeader(key, headers.get(key)); } } client.executeMethod(method); return method.getResponseBodyAsString(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String get(String url, Map headers, ProxyIP ipInfo) { HttpClient client = new HttpClient(); if (ipInfo != null) client.getHostConfiguration().setProxy(ipInfo.getIp(), ipInfo.getPort()); client.getHttpConnectionManager().getParams().setConnectionTimeout(5000); try { GetMethod method = new GetMethod(url); if (headers != null) { Iterator keys = headers.keySet().iterator(); while (keys.hasNext()) { String key = keys.next(); method.addRequestHeader(key, headers.get(key)); } } client.executeMethod(method); return method.getResponseBodyAsString(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String get(String url, Map headers, ProxyIP ipInfo, String userName, String proxyPwd) { CloseableHttpClient httpclient = null; if (!StringUtil.isNullOrEmpty(userName)) { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(ipInfo.getIp(), ipInfo.getPort()), new UsernamePasswordCredentials(userName, proxyPwd)); httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).build(); } else HttpClients.createDefault(); HttpHost proxy = null; if (ipInfo.getPort() != 0 && ipInfo.getPort() != 80) { proxy = new HttpHost(ipInfo.getIp(), ipInfo.getPort()); } else proxy = new HttpHost(ipInfo.getIp()); RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); HttpPost post = new HttpPost(url); try { post.setConfig(config); CloseableHttpResponse response = httpclient.execute(post); return EntityUtils.toString(response.getEntity()); } catch (Exception e) { return null; } } public static String getAsString(String url, String fromCharset, String toCharset) { HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); try { client.executeMethod(method); String response = method.getResponseBodyAsString(); String s1 = new String(response.getBytes(fromCharset), toCharset); return s1; } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String get(String url, Map params, Map headers) { HttpClient client = new HttpClient(); try { Iterator keys = params.keySet().iterator(); url += "?"; while (keys.hasNext()) { String key = keys.next(); url += String.format("%s=%s&", key, URLEncoder.encode(params.get(key), "UTF-8")); } GetMethod method = new GetMethod(url); if (headers != null) { keys = headers.keySet().iterator(); while (keys.hasNext()) { String key = keys.next(); headers.get(key); method.setRequestHeader(key, headers.get(key)); } } client.executeMethod(method); System.out.println(method.getResponseHeader("cookies")); String result = method.getResponseBodyAsString(); return result; } catch (Exception e) { e.printStackTrace(); } return ""; } public static InputStream getAsInputStream(String url) { HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); try { client.executeMethod(method); return method.getResponseBodyAsStream(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return null; } /** * 下载图片文件 * * @param url * @return */ public static InputStream downLoadImg(String url) throws org.yeshi.utils.exception.HttpException { HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); try { client.executeMethod(method); Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader == null || contentTypeHeader.getValue().contains("text")) throw new org.yeshi.utils.exception.HttpException(2, "图片下载出错"); return method.getResponseBodyAsStream(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return null; } public static InputStream getAsInputStream(String url, Map headers) { HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); Iterator keys = headers.keySet().iterator(); while (keys.hasNext()) { String key = keys.next(); method.setRequestHeader(key, headers.get(key)); } try { client.executeMethod(method); return method.getResponseBodyAsStream(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return null; } public static InputStream getImage(String url) { HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); try { method.setRequestHeader("Content-Type", "image/jpeg"); client.executeMethod(method); return method.getResponseBodyAsStream(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return null; } public static String postSimple(String url) { HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); try { client.executeMethod(method); return method.getResponseBodyAsString(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String post(String url) { HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); method.addRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestHeader("Content-Type", "text/html;charset=UTF-8"); try { client.executeMethod(method); return convertInputStreamToString(method.getResponseBodyAsStream()); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String post(String url, Map map) { Iterator its = map.keySet().iterator(); NameValuePair[] params = new NameValuePair[map.keySet().size()]; int p = 0; while (its.hasNext()) { String key = its.next(); NameValuePair np = new NameValuePair(key, map.get(key)); params[p] = np; p++; } HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); method.addRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestBody(params); try { client.executeMethod(method); return convertInputStreamToString(method.getResponseBodyAsStream()); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String post(String url, Map map, Map headers, ProxyIP ipInfo) { HttpClient client = new HttpClient(); if (ipInfo != null) client.getHostConfiguration().setProxy(ipInfo.getIp(), ipInfo.getPort()); PostMethod pm = new PostMethod(url);// 创建HttpPost对象 NameValuePair[] ns = new NameValuePair[map.keySet().size()]; Iterator its = map.keySet().iterator(); int index = 0; while (its.hasNext()) { String key = its.next(); ns[index++] = (new NameValuePair(key, map.get(key))); } if (headers != null) { its = headers.keySet().iterator(); while (its.hasNext()) { String key = its.next(); pm.setRequestHeader(key, headers.get(key)); } } try { pm.setRequestBody(ns); client.executeMethod(pm); return pm.getResponseBodyAsString(); } catch (Exception e) { e.printStackTrace(); } return ""; } public static String post(String url, Map map, Map headers) { HttpClient client = new HttpClient(); // client.getHostConfiguration().setProxy("192.168.3.122", 8888); PostMethod pm = new PostMethod(url);// 创建HttpPost对象 NameValuePair[] ns = new NameValuePair[map.keySet().size()]; Iterator its = map.keySet().iterator(); int index = 0; while (its.hasNext()) { String key = its.next(); ns[index++] = (new NameValuePair(key, map.get(key))); } if (headers != null) { its = headers.keySet().iterator(); while (its.hasNext()) { String key = its.next(); pm.setRequestHeader(key, headers.get(key)); } } try { pm.setRequestBody(ns); client.executeMethod(pm); return pm.getResponseBodyAsString(); } catch (Exception e) { e.printStackTrace(); } return ""; } private static String convertInputStreamToString(InputStream inputStream) { BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); StringBuffer stringBuffer = new StringBuffer(); String str = ""; try { while ((str = br.readLine()) != null) { stringBuffer.append(str); } } catch (IOException e) { e.printStackTrace(); } return stringBuffer.toString(); } @SuppressWarnings("deprecation") public static String post(String url, String entity) { HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); method.addRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestBody(entity); try { client.executeMethod(method); return convertInputStreamToString(method.getResponseBodyAsStream()); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } @SuppressWarnings("deprecation") public static String post(String url, String entity, String charset) { HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); method.addRequestHeader("Content-Type", "text/html;charset=" + charset); method.setRequestHeader("Content-Type", "text/html;charset=" + charset); method.setRequestBody(entity); try { client.executeMethod(method); return convertInputStreamToString(method.getResponseBodyAsStream()); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static String postInputStream(String url, File file) throws FileNotFoundException { HttpClient client = new HttpClient(); PostMethod postMethod = new PostMethod(url); Part[] parts = {new StringPart("filename", file.getName()), new StringPart("filelength", file.length() + ""), new StringPart("content-type", "image/jpg"), new FilePart("file", file)}; postMethod.setRequestEntity(new MultipartRequestEntity(parts, postMethod.getParams())); try { client.executeMethod(postMethod); return convertInputStreamToString(postMethod.getResponseBodyAsStream()); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return ""; } public static File downloadFile(String url, String targetPath) { try { URL httpurl = new URL(url); FileUtils.copyURLToFile(httpurl, new File(targetPath)); } catch (Exception e) { e.printStackTrace(); } File f = new File(targetPath); if (!f.exists() || f.length() <= 0) return null; return f; } @SuppressWarnings("deprecation") public static InputStream postForInputstream(String url, String entity) { HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); method.addRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestHeader("Content-Type", "text/html;charset=UTF-8"); method.setRequestBody(entity); try { client.executeMethod(method); return method.getResponseBodyAsStream(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return null; } public static String post(String url, String fileKey, File f) { PostMethod filePost = new PostMethod(url); HttpClient client = new HttpClient(); try { Part[] parts = {new FilePart(fileKey, f)}; filePost.setRequestEntity(new MultipartRequestEntity(parts, filePost.getParams())); client.getHttpConnectionManager().getParams().setConnectionTimeout(5000); int status = client.executeMethod(filePost); if (status == HttpStatus.SC_OK) { return filePost.getResponseBodyAsString(); } else { System.out.println("上传失败"); } } catch (Exception ex) { ex.printStackTrace(); } finally { filePost.releaseConnection(); } return null; } @SuppressWarnings("deprecation") public static String httpsPost(String url, String body, String pwd, InputStream cert) throws Exception { String result = ""; KeyStore keyStore = KeyStore.getInstance("PKCS12"); InputStream instream = cert; try { keyStore.load(instream, pwd.toCharArray()); } finally { instream.close(); } // Trust own CA and all self-signed certs SSLContext sslcontext = SSLContexts.custom().loadKeyMaterial(keyStore, pwd.toCharArray()).build(); // Allow TLSv1 protocol only SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, new String[]{"TLSv1"}, null, SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER); CloseableHttpClient httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).build(); try { HttpPost httppost = new HttpPost(url); httppost.setHeader("Content-Type", "text/html;charset=UTF-8"); System.out.println("executing request" + httppost.getRequestLine()); httppost.setEntity(new StringEntity(body, "UTF-8")); CloseableHttpResponse response = httpclient.execute(httppost); try { HttpEntity entity = response.getEntity(); if (entity != null) { result = convertInputStreamToString(entity.getContent()); } EntityUtils.consume(entity); } finally { response.close(); } } finally { httpclient.close(); } return result; } /** * 获取post返回location * * @param url * @return */ public static String getLocation(String url) { HttpClient client = new HttpClient(); PostMethod method = new PostMethod(url); try { client.executeMethod(method); return method.getResponseHeader("location").getValue(); } catch (HttpException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return null; } }