using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Security; using System.Reflection; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Web; namespace Pcb.Common { /// /// HTTP请求帮助类 /// public class HttpRequestHelper { /// /// Post请求 /// /// 请求地址 /// 请求参数 /// /// 流信息 public static string DoPost(string url, string postParam, string encode = "utf-8", string contentType = "application/x-www-form-urlencoded", int timeout = 100000, string authorization = "", bool isLog = false) { try { ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult; var param = Encoding.GetEncoding(encode).GetBytes(postParam); HttpWebRequest req = null; if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase)) { ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult; req = WebRequest.Create(url) as HttpWebRequest; req.ProtocolVersion = HttpVersion.Version10; } else { req = WebRequest.Create(url) as HttpWebRequest; } req.Method = "POST"; req.ContentType = contentType; req.ContentLength = param.Length; req.KeepAlive = false; req.Timeout = timeout; if (!string.IsNullOrEmpty(authorization)) { req.Headers[HttpRequestHeader.Authorization] = authorization; } req.ProtocolVersion = HttpVersion.Version10; using (var reqstream = req.GetRequestStream()) { reqstream.Write(param, 0, param.Length); } using (var response = (HttpWebResponse)req.GetResponse()) { var stream = response.GetResponseStream(); if (stream == null) return string.Empty; var reader = new StreamReader(stream); var data = reader.ReadToEnd(); if (isLog) { Log(url, postParam, data); } return data; } } catch (Exception ex) { if (isLog) { Log(url, postParam, ex); } LogHelper.Error(ex.ToString()); } return ""; } /// /// 处理GET请求 /// /// 请求地址 /// 请求参数 /// 流信息 public static string DoGet(string url, string getParam, int timeout = 500000, bool isLog = false) { try { System.GC.Collect(); System.Net.ServicePointManager.DefaultConnectionLimit = 500; if (!string.IsNullOrWhiteSpace(getParam)) { url = url + "?" + getParam; } var req = WebRequest.Create(url) as HttpWebRequest; req.Method = "Get"; req.KeepAlive = false; req.Timeout = timeout; using (var response = (HttpWebResponse)req.GetResponse()) { var stream = response.GetResponseStream(); if (stream == null) return string.Empty; var reader = new StreamReader(stream); var data = reader.ReadToEnd(); response.Close(); if (isLog) { Log(url, getParam, data); } return data; } } catch (Exception ex) { if (isLog) { Log(url, getParam, ex); } LogHelper.Error(ex.ToString()); } return ""; } public static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) { // 总是接受 return true; } public static string HttpPost(string url, string data, IDictionary headers = null) { var result = ""; using (var client = new HttpClient()) { var values = new List>(); var dList = data.Split('&').ToList(); foreach (var item in dList) { var arr = item.Split('=').ToArray(); values.Add(new KeyValuePair(arr[0], (arr[1]))); } var content = new FormUrlEncodedContent(values); var response = client.PostAsync(url, content); result = response.Result.Content.ReadAsStringAsync().Result; } return result; } /// /// 请求操作 /// /// /// /// /// /// /// /// /// /// /// public static string HttpRequest(string apikey, string apisec, string method, string url, Dictionary urlparams = null, Dictionary data = null, Dictionary headers = null, int timeout = 10, bool urlsign = true) { string responseText = ""; if (urlsign) { if (urlparams == null) { urlparams = new Dictionary(); } if (!urlparams.ContainsKey("appid")) { urlparams.Add("appid", apikey); } if (!urlparams.ContainsKey("timestamp")) { urlparams.Add("timestamp", DateTime.Now.ToTimestamp() + ""); } Dictionary signparams = urlparams; if (data != null) { signparams = signparams.Union(data).ToDictionary(k => k.Key, v => v.Value); } if (urlparams.ContainsKey("signature")) { urlparams["signature"] = Extensions.GenSign(signparams, apisec); } else { urlparams.Add("signature", Extensions.GenSign(signparams, apisec)); } } if (url.Length == 0) { return null; } byte[] byteArray = null; string urlparam = Extensions.Paramjoin(urlparams); if (urlparam.Length > 0) { if (url.IndexOf("?") > 0) { url += urlparam; } else { url += "?" + urlparam; } } if (data != null) { byteArray = Encoding.UTF8.GetBytes(Extensions.Paramjoin(data)); } HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.ProtocolVersion = HttpVersion.Version10; request.Timeout = timeout * 1000; request.Method = method.ToUpper(); request.KeepAlive = false; if (byteArray != null && byteArray.Length > 0) { request.ContentLength = byteArray.Length; } if (headers != null && headers.Count > 0) { foreach (KeyValuePair entry in headers) { request.Headers.Add(entry.Key, entry.Value); } } request.ContentType = "application/x-www-form-urlencoded"; if (byteArray != null && byteArray.Length > 0) { Stream dataStream = request.GetRequestStream(); dataStream.Write(byteArray, 0, byteArray.Length); dataStream.Close(); dataStream.Dispose(); } int httpStatusCode = 200; HttpWebResponse response = null; try { response = (HttpWebResponse)request.GetResponse(); httpStatusCode = (int)response.StatusCode; } catch (WebException e) { response = (HttpWebResponse)e.Response; if (response != null) { httpStatusCode = (int)response.StatusCode; } else { return e.Message; } } // Console.WriteLine(httpStatusCode); Stream resStream = response.GetResponseStream(); if (resStream == null) { return ""; } try { StreamReader reader = new StreamReader(resStream); responseText = reader.ReadToEnd(); reader.Close(); response.Close(); } catch (ArgumentException) { } responseText = HttpUtility.UrlDecode(responseText); return responseText; } #region 用于查错 add by daiyanbing 2020年3月12日 10:19:43 static void Log(string Url, string Params, string Result) { LogHelper.Info("请求网址:" + Url + ",参数:" + Params + ",结果:" + Result); } static void Log(string Url, string Params, Exception Except) { LogHelper.Info("请求网址:" + Url + ",参数:" + Params + ",报错:" + Except.ToString()); } #endregion } public class HttpContentType { public const string application_json = "application/json"; public const string text_plain = "text/plain"; } public class CaptureHelper { #region 抓取页面代码 static Encoding encoding = Encoding.UTF8; public string[] HttpRequestHeader = { "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+SE+2.X+MetaSr+1.0)", "Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+.NET+CLR+2.0.50727;+360SE)", "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727)", "Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+WOW64;+Trident/4.0;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729)", "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+QQDownload+708;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+3.0.4506.2152;+.NET+CLR+3.5.30729;+CIBA;+360SE)", "Mozilla/5.0+(Windows+NT+6.1)+AppleWebKit/535.12+(KHTML,+like+Gecko)+Maxthon/3.3.8.3000+Chrome/18.0.966.0+Safari/535.12", "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1)", "Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+1.1.4322)" }; static void ForceCanonicalPathAndQuery(Uri uri) { string paq = uri.PathAndQuery; // need to access PathAndQuery FieldInfo flagsFieldInfo = typeof(Uri).GetField("m_Flags", BindingFlags.Instance | BindingFlags.NonPublic); ulong flags = (ulong)flagsFieldInfo.GetValue(uri); flags &= ~((ulong)0x30); // Flags.PathNotCanonical|Flags.QueryNotCanonical flagsFieldInfo.SetValue(uri, flags); } public string GetHtml(string url, int timeout) { string content = string.Empty; try { Uri uri = new Uri(url); ForceCanonicalPathAndQuery(uri); HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(uri); //WebProxy proxy = new WebProxy(); //定義一個網關對象 //proxy.Address = new Uri("http://proxy.domain.com:3128"); //網關服務器:端口 //proxy.Credentials = new NetworkCredential("f3210316", "6978233"); //用戶名,密碼 //hwr.UseDefaultCredentials = true; //啟用網關認証 //hwr.Proxy = proxy; hwr.ContentType = "text/html; charset=UTF-8"; hwr.Accept = "*/*";// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; // hwr.Headers["Accept-Language"] = "zh-cn,zh;q=0.5"; hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36";// HttpRequestHeader[new Random().Next(0, 8)]; hwr.Method = "GET"; hwr.Timeout = timeout; //Encoding encoding = null; //try //{ // encoding = Encoding.GetEncoding(DecodeData(url)); //} //catch //{ // encoding = Encoding.GetEncoding("utf-8"); //} HttpWebResponse hwrs = null; hwrs = (HttpWebResponse)hwr.GetResponse(); if (hwrs.StatusCode == HttpStatusCode.OK) { Stream s = hwrs.GetResponseStream(); StreamReader sr = new StreamReader(s, encoding); content = sr.ReadToEnd(); s.Close(); sr.Close(); } else { content = "404"; } } catch (Exception ex) { content = ex.Message.ToString();// "404"; //LogHelper.Write(this.Name,ex.ToString()); //return string.Empty; } return content;//.Replace("\r", "").Replace("\n", "").Replace("\t", ""); } public string GetHtml(string url, int timeout, Encoding encode) { string content = string.Empty; for (int i = 0; i < 3; i++) { try { Uri uri = new Uri(url); HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(uri); hwr.ContentType = "application/x-www-form-urlencoded"; hwr.UserAgent = HttpRequestHeader[new Random().Next(0, 6)]; //"Mozilla/5.0 (compatible; +Googlebot/2.1;++http://www.google.com/bot.html)"; //hwr.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.01; Windows NT 5.0)"; hwr.Method = "GET"; hwr.Timeout = timeout; //Encoding encoding = null; //try //{ // encoding = Encoding.GetEncoding(DecodeData(url)); //} //catch //{ // encoding = Encoding.GetEncoding("utf-8"); //} HttpWebResponse hwrs = null; hwrs = (HttpWebResponse)hwr.GetResponse(); Stream s = hwrs.GetResponseStream(); StreamReader sr = new StreamReader(s, encode); content = sr.ReadToEnd(); s.Close(); sr.Close(); break; } catch (Exception ex) { throw ex; Thread.Sleep(3000); //LogHelper.Write(this.Name,ex.ToString()); //return string.Empty; } } return content;//.Replace("\r", "").Replace("\n", "").Replace("\t", ""); } public string GetPageToStr(string uri, string encoding) { /// /// 取得远程web源文件 /// /// 远程文件地址 /// 编码格式 /// 返回string类型 string strHtml; StreamReader sr = null; //用来读取流 StreamWriter sw = null; //用来写文件 Encoding code = Encoding.GetEncoding(encoding); //定义编码 //构造web请求,发送请求,获取响应 WebRequest HttpWebRequest = null; WebResponse HttpWebResponse = null; HttpWebRequest = WebRequest.Create(uri); HttpWebResponse = HttpWebRequest.GetResponse(); //获得流 sr = new StreamReader(HttpWebResponse.GetResponseStream(), code); strHtml = sr.ReadToEnd(); return strHtml; } public string GetHtml(string url) { return GetHtml(url, 60 * 1000); } /// /// 有三次请求机会 /// /// /// public string GetHtmlTry3times(string url) { int i = 0; while (i < 3) { try { return GetHtml(url); } catch { i++; Thread.Sleep(1000); } } return ""; } public string GetHtml(string url, Encoding encode) { return GetHtml(url, 60 * 1000, encode); } /// /// 有三次请求机会 /// /// /// public string GetHtmlTry3times(string url, Encoding encode) { int i = 0; while (i < 3) { try { string html = GetHtml(url, encode); if (html == "") { i++; Thread.Sleep(1000); } else { return html; } } catch { i++; Thread.Sleep(1000); } } return ""; } #endregion } }