using System;
|
using System.Collections.Generic;
|
using System.IO;
|
using System.Linq;
|
using System.Net;
|
using System.Net.Http;
|
using System.Net.Security;
|
using System.Reflection;
|
using System.Security.Cryptography.X509Certificates;
|
using System.Text;
|
using System.Threading;
|
using System.Threading.Tasks;
|
using System.Web;
|
|
namespace Pcb.Common
|
{
|
/// <summary>
|
/// HTTP请求帮助类
|
/// </summary>
|
public class HttpRequestHelper
|
{
|
/// <summary>
|
/// Post请求
|
/// </summary>
|
/// <param name="url">请求地址</param>
|
/// <param name="postParam">请求参数</param>
|
/// <param name="contentType"></param>
|
/// <returns>流信息</returns>
|
public static string DoPost(string url, string postParam, string encode = "utf-8", string contentType = "application/x-www-form-urlencoded", int timeout = 100000, string authorization = "", bool isLog = false)
|
{
|
try
|
{
|
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
|
var param = Encoding.GetEncoding(encode).GetBytes(postParam);
|
HttpWebRequest req = null;
|
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
|
{
|
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
|
req = WebRequest.Create(url) as HttpWebRequest;
|
req.ProtocolVersion = HttpVersion.Version10;
|
}
|
else
|
{
|
req = WebRequest.Create(url) as HttpWebRequest;
|
}
|
|
req.Method = "POST";
|
req.ContentType = contentType;
|
req.ContentLength = param.Length;
|
req.KeepAlive = false;
|
req.Timeout = timeout;
|
if (!string.IsNullOrEmpty(authorization))
|
{
|
req.Headers[HttpRequestHeader.Authorization] = authorization;
|
}
|
|
req.ProtocolVersion = HttpVersion.Version10;
|
using (var reqstream = req.GetRequestStream())
|
{
|
reqstream.Write(param, 0, param.Length);
|
}
|
using (var response = (HttpWebResponse)req.GetResponse())
|
{
|
var stream = response.GetResponseStream();
|
if (stream == null)
|
return string.Empty;
|
var reader = new StreamReader(stream);
|
var data = reader.ReadToEnd();
|
if (isLog) { Log(url, postParam, data); }
|
return data;
|
}
|
}
|
catch (Exception ex)
|
{
|
if (isLog) { Log(url, postParam, ex); }
|
LogHelper.Error(ex.ToString());
|
}
|
return "";
|
}
|
|
/// <summary>
|
/// 处理GET请求
|
/// </summary>
|
/// <param name="url">请求地址</param>
|
/// <param name="getParam">请求参数</param>
|
/// <returns>流信息</returns>
|
public static string DoGet(string url, string getParam, int timeout = 500000, bool isLog = false)
|
{
|
try
|
{
|
System.GC.Collect();
|
System.Net.ServicePointManager.DefaultConnectionLimit = 500;
|
if (!string.IsNullOrWhiteSpace(getParam))
|
{ url = url + "?" + getParam; }
|
var req = WebRequest.Create(url) as HttpWebRequest;
|
req.Method = "Get";
|
req.KeepAlive = false;
|
req.Timeout = timeout;
|
using (var response = (HttpWebResponse)req.GetResponse())
|
{
|
var stream = response.GetResponseStream();
|
if (stream == null)
|
return string.Empty;
|
var reader = new StreamReader(stream);
|
var data = reader.ReadToEnd();
|
response.Close();
|
if (isLog) { Log(url, getParam, data); }
|
return data;
|
}
|
}
|
catch (Exception ex)
|
{
|
if (isLog) { Log(url, getParam, ex); }
|
LogHelper.Error(ex.ToString());
|
}
|
return "";
|
}
|
|
public static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
|
{
|
// 总是接受
|
return true;
|
}
|
|
|
public static string HttpPost(string url, string data, IDictionary<object, string> headers = null)
|
{
|
var result = "";
|
using (var client = new HttpClient())
|
{
|
var values = new List<KeyValuePair<string, string>>();
|
var dList = data.Split('&').ToList();
|
foreach (var item in dList)
|
{
|
var arr = item.Split('=').ToArray();
|
values.Add(new KeyValuePair<string, string>(arr[0], (arr[1])));
|
}
|
|
|
var content = new FormUrlEncodedContent(values);
|
|
var response = client.PostAsync(url, content);
|
|
result = response.Result.Content.ReadAsStringAsync().Result;
|
}
|
return result;
|
}
|
|
/// <summary>
|
/// 请求操作
|
/// </summary>
|
/// <param name="apikey"></param>
|
/// <param name="apisec"></param>
|
/// <param name="method"></param>
|
/// <param name="url"></param>
|
/// <param name="urlparams"></param>
|
/// <param name="data"></param>
|
/// <param name="headers"></param>
|
/// <param name="timeout"></param>
|
/// <param name="urlsign"></param>
|
/// <returns></returns>
|
public static string HttpRequest(string apikey, string apisec, string method, string url, Dictionary<string, string> urlparams = null, Dictionary<string, string> data = null, Dictionary<string, string> headers = null, int timeout = 10, bool urlsign = true)
|
{
|
string responseText = "";
|
if (urlsign)
|
{
|
if (urlparams == null)
|
{
|
urlparams = new Dictionary<string, string>();
|
}
|
if (!urlparams.ContainsKey("appid"))
|
{
|
urlparams.Add("appid", apikey);
|
}
|
if (!urlparams.ContainsKey("timestamp"))
|
{
|
urlparams.Add("timestamp", DateTime.Now.ToTimestamp() + "");
|
}
|
|
Dictionary<string, string> signparams = urlparams;
|
if (data != null)
|
{
|
signparams = signparams.Union(data).ToDictionary(k => k.Key, v => v.Value);
|
}
|
if (urlparams.ContainsKey("signature"))
|
{
|
urlparams["signature"] = Extensions.GenSign(signparams, apisec);
|
}
|
else
|
{
|
urlparams.Add("signature", Extensions.GenSign(signparams, apisec));
|
}
|
}
|
|
if (url.Length == 0)
|
{
|
return null;
|
}
|
byte[] byteArray = null;
|
string urlparam = Extensions.Paramjoin(urlparams);
|
if (urlparam.Length > 0)
|
{
|
if (url.IndexOf("?") > 0)
|
{
|
url += urlparam;
|
}
|
else
|
{
|
url += "?" + urlparam;
|
}
|
}
|
|
if (data != null)
|
{
|
byteArray = Encoding.UTF8.GetBytes(Extensions.Paramjoin(data));
|
}
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
request.ProtocolVersion = HttpVersion.Version10;
|
request.Timeout = timeout * 1000;
|
request.Method = method.ToUpper();
|
request.KeepAlive = false;
|
if (byteArray != null && byteArray.Length > 0)
|
{
|
request.ContentLength = byteArray.Length;
|
}
|
|
if (headers != null && headers.Count > 0)
|
{
|
foreach (KeyValuePair<string, string> entry in headers)
|
{
|
request.Headers.Add(entry.Key, entry.Value);
|
}
|
}
|
|
request.ContentType = "application/x-www-form-urlencoded";
|
|
if (byteArray != null && byteArray.Length > 0)
|
{
|
Stream dataStream = request.GetRequestStream();
|
dataStream.Write(byteArray, 0, byteArray.Length);
|
dataStream.Close();
|
dataStream.Dispose();
|
}
|
int httpStatusCode = 200;
|
HttpWebResponse response = null;
|
try
|
{
|
response = (HttpWebResponse)request.GetResponse();
|
httpStatusCode = (int)response.StatusCode;
|
}
|
catch (WebException e)
|
{
|
response = (HttpWebResponse)e.Response;
|
if (response != null)
|
{
|
|
httpStatusCode = (int)response.StatusCode;
|
}
|
else
|
{
|
return e.Message;
|
}
|
|
}
|
// Console.WriteLine(httpStatusCode);
|
Stream resStream = response.GetResponseStream();
|
if (resStream == null)
|
{
|
return "";
|
}
|
|
try
|
{
|
StreamReader reader = new StreamReader(resStream);
|
responseText = reader.ReadToEnd();
|
reader.Close();
|
response.Close();
|
}
|
catch (ArgumentException)
|
{
|
}
|
responseText = HttpUtility.UrlDecode(responseText);
|
return responseText;
|
}
|
|
#region 用于查错 add by daiyanbing 2020年3月12日 10:19:43
|
static void Log(string Url, string Params, string Result)
|
{
|
LogHelper.Info("请求网址：" + Url + ",参数：" + Params + ",结果：" + Result);
|
}
|
static void Log(string Url, string Params, Exception Except)
|
{
|
LogHelper.Info("请求网址：" + Url + ",参数：" + Params + ",报错：" + Except.ToString());
|
}
|
#endregion
|
}
|
|
public class HttpContentType
|
{
|
public const string application_json = "application/json";
|
public const string text_plain = "text/plain";
|
}
|
|
|
public class CaptureHelper
|
{
|
#region 抓取页面代码
|
static Encoding encoding = Encoding.UTF8;
|
public string[] HttpRequestHeader = { "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+SE+2.X+MetaSr+1.0)",
|
"Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+.NET+CLR+2.0.50727;+360SE)",
|
"Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727)",
|
"Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+WOW64;+Trident/4.0;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729)",
|
"Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+QQDownload+708;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+3.0.4506.2152;+.NET+CLR+3.5.30729;+CIBA;+360SE)",
|
"Mozilla/5.0+(Windows+NT+6.1)+AppleWebKit/535.12+(KHTML,+like+Gecko)+Maxthon/3.3.8.3000+Chrome/18.0.966.0+Safari/535.12",
|
"Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1)",
|
"Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+1.1.4322)" };
|
|
static void ForceCanonicalPathAndQuery(Uri uri)
|
{
|
|
string paq = uri.PathAndQuery; // need to access PathAndQuery
|
|
FieldInfo flagsFieldInfo = typeof(Uri).GetField("m_Flags", BindingFlags.Instance | BindingFlags.NonPublic);
|
|
ulong flags = (ulong)flagsFieldInfo.GetValue(uri);
|
|
flags &= ~((ulong)0x30); // Flags.PathNotCanonical|Flags.QueryNotCanonical
|
|
flagsFieldInfo.SetValue(uri, flags);
|
|
}
|
|
public string GetHtml(string url, int timeout)
|
{
|
string content = string.Empty;
|
try
|
{
|
Uri uri = new Uri(url);
|
ForceCanonicalPathAndQuery(uri);
|
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(uri);
|
|
//WebProxy proxy = new WebProxy(); //定義一個網關對象
|
//proxy.Address = new Uri("http://proxy.domain.com:3128"); //網關服務器:端口
|
//proxy.Credentials = new NetworkCredential("f3210316", "6978233"); //用戶名,密碼
|
//hwr.UseDefaultCredentials = true; //啟用網關認証
|
//hwr.Proxy = proxy;
|
|
hwr.ContentType = "text/html; charset=UTF-8";
|
hwr.Accept = "*/*";// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
// hwr.Headers["Accept-Language"] = "zh-cn,zh;q=0.5";
|
hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36";// HttpRequestHeader[new Random().Next(0, 8)];
|
hwr.Method = "GET";
|
hwr.Timeout = timeout;
|
//Encoding encoding = null;
|
//try
|
//{
|
// encoding = Encoding.GetEncoding(DecodeData(url));
|
//}
|
//catch
|
//{
|
// encoding = Encoding.GetEncoding("utf-8");
|
|
//}
|
HttpWebResponse hwrs = null;
|
|
hwrs = (HttpWebResponse)hwr.GetResponse();
|
if (hwrs.StatusCode == HttpStatusCode.OK)
|
{
|
Stream s = hwrs.GetResponseStream();
|
StreamReader sr = new StreamReader(s, encoding);
|
content = sr.ReadToEnd();
|
s.Close();
|
sr.Close();
|
}
|
else
|
{
|
content = "404";
|
}
|
}
|
catch (Exception ex)
|
{
|
content = ex.Message.ToString();// "404";
|
//LogHelper.Write(this.Name,ex.ToString());
|
//return string.Empty;
|
}
|
return content;//.Replace("\r", "").Replace("\n", "").Replace("\t", "");
|
}
|
public string GetHtml(string url, int timeout, Encoding encode)
|
{
|
string content = string.Empty;
|
for (int i = 0; i < 3; i++)
|
{
|
try
|
{
|
Uri uri = new Uri(url);
|
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(uri);
|
hwr.ContentType = "application/x-www-form-urlencoded";
|
hwr.UserAgent = HttpRequestHeader[new Random().Next(0, 6)];
|
//"Mozilla/5.0 (compatible; +Googlebot/2.1;++http://www.google.com/bot.html)";
|
//hwr.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.01; Windows NT 5.0)";
|
hwr.Method = "GET";
|
hwr.Timeout = timeout;
|
//Encoding encoding = null;
|
//try
|
//{
|
// encoding = Encoding.GetEncoding(DecodeData(url));
|
//}
|
//catch
|
//{
|
// encoding = Encoding.GetEncoding("utf-8");
|
|
//}
|
HttpWebResponse hwrs = null;
|
|
hwrs = (HttpWebResponse)hwr.GetResponse();
|
|
Stream s = hwrs.GetResponseStream();
|
StreamReader sr = new StreamReader(s, encode);
|
content = sr.ReadToEnd();
|
s.Close();
|
sr.Close();
|
break;
|
}
|
catch (Exception ex)
|
{
|
throw ex;
|
Thread.Sleep(3000);
|
//LogHelper.Write(this.Name,ex.ToString());
|
//return string.Empty;
|
}
|
|
}
|
return content;//.Replace("\r", "").Replace("\n", "").Replace("\t", "");
|
}
|
public string GetPageToStr(string uri, string encoding)
|
{
|
///
|
/// 取得远程web源文件
|
///
|
/// 远程文件地址
|
/// 编码格式
|
/// 返回string类型
|
string strHtml;
|
StreamReader sr = null; //用来读取流
|
StreamWriter sw = null; //用来写文件
|
Encoding code = Encoding.GetEncoding(encoding); //定义编码
|
|
//构造web请求，发送请求，获取响应
|
WebRequest HttpWebRequest = null;
|
WebResponse HttpWebResponse = null;
|
HttpWebRequest = WebRequest.Create(uri);
|
HttpWebResponse = HttpWebRequest.GetResponse();
|
|
//获得流
|
sr = new StreamReader(HttpWebResponse.GetResponseStream(), code);
|
strHtml = sr.ReadToEnd();
|
return strHtml;
|
}
|
|
public string GetHtml(string url)
|
{
|
return GetHtml(url, 60 * 1000);
|
}
|
|
/// <summary>
|
/// 有三次请求机会
|
/// </summary>
|
/// <param name="url"></param>
|
/// <returns></returns>
|
public string GetHtmlTry3times(string url)
|
{
|
int i = 0;
|
while (i < 3)
|
{
|
try
|
{
|
return GetHtml(url);
|
}
|
catch
|
{
|
i++;
|
Thread.Sleep(1000);
|
}
|
}
|
|
return "";
|
}
|
|
public string GetHtml(string url, Encoding encode)
|
{
|
return GetHtml(url, 60 * 1000, encode);
|
}
|
|
/// <summary>
|
/// 有三次请求机会
|
/// </summary>
|
/// <param name="url"></param>
|
/// <returns></returns>
|
public string GetHtmlTry3times(string url, Encoding encode)
|
{
|
int i = 0;
|
while (i < 3)
|
{
|
try
|
{
|
string html = GetHtml(url, encode);
|
if (html == "")
|
{
|
i++;
|
Thread.Sleep(1000);
|
}
|
else
|
{
|
return html;
|
}
|
}
|
catch
|
{
|
i++;
|
Thread.Sleep(1000);
|
}
|
}
|
|
return "";
|
}
|
#endregion
|
}
|
}
|
