Java、C#双语版HttpHelper类(解决网页抓取乱码问题)(二)
(input != null && !input.equals("")) {
Matcher matcher = pattern.matcher(input);
if (matcher.find()) {
return matcher.group(1);
}
}
return null;
}
private static InputStream getInputStream(HttpURLConnection conn) throws Exception {
String ContentEncoding = conn.getHeaderField("Content-Encoding");
if (ContentEncoding != null) {
ContentEncoding = ContentEncoding.toLowerCase();
if (ContentEncoding.indexOf("gzip") != 1)
return new GZIPInputStream(conn.getInputStream());
else if (ContentEncoding.indexOf("deflate") != 1)
return new DeflaterInputStream(conn.getInputStream());
}
return conn.getInputStream();
}
static HttpURLConnection getConnection(String url, String method, String param, Map header) throws Exception {
HttpURLConnection conn = (HttpURLConnection) (new URL(url)).openConnection();
conn.setRequestMethod(method);
// 设置通用的请求属性
conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36");
conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
String ContentEncoding = null;
if (header != null) {
for (Entry entry : header.entrySet()) {
if (entry.getKey().equalsIgnoreCase("Content-Encoding"))
ContentEncoding = entry.getValue();
conn.setRequestProperty(entry.getKey(), entry.getValue());
}
}
if (method == "POST") {
conn.setDoOutput(true);
conn.setDoInput(true);
if (param != null && !param.equals("")) {
OutputStream output = conn.getOutputStream();
if (ContentEncoding != null) {
if (ContentEncoding.indexOf("gzip") > 0) {
output=new GZIPOutputStream(output);
}
else if(ContentEncoding.indexOf("deflate") > 0) {
output=new DeflaterOutputStream(output);
}
}
output.write(param.getBytes());
}
}
// 建立实际的连接
conn.connect();
return conn;
}
}
复制代码
C#实现
复制代码
using System;
using System.Collections;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.IO.Compression;
using System.Collections.Generic;
using System.Collections.Specialized;
namespace CSharp.Util.Net
{
public class HttpHelper
{
private static bool RemoteCertificateva lidate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
//用户https请求
return true; //总是接受
}
public static string SendPost(string url, string data)
{
return Send(url, "POST", null, null);