C# 关于爬取网站数据遇到csrf-token的分析与解决

内容导读

互联网集市收集整理的这篇技术教程文章主要介绍了C# 关于爬取网站数据遇到csrf-token的分析与解决，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含5027字，纯文字阅读大概需要8分钟。

内容图文

需求
某航空公司物流单信息查询，是一个post请求。通过后台模拟POST HTTP请求发现无法获取页面数据，通过查看航空公司网站后，发现网站使用避免CSRF攻击机制，直接发挥40X错误。

关于CSRF
读者自行百度

网站HTTP请求分析
Headers

Form Data

在head里包含了cookie 与 x-csrf-token formdata 里包含了_csrf （与head里的值是一样的）.

这里通过查看该网站的JS源代码发现_csrf 来自于网c#教程页的head标签里

猜测cookie与 x-csrf-token是有一定的有效期，并且他们共同作用来防御CSRF攻击。

解决方案
1，首先请求一下该航空公司的网站，获取cookie与_csrf

2，然后C# 模拟http分别在head和formdata里加入如上参数，发起请求

代码

public class CSRFToken
 {
 string cookie;//用于请求的站点的cookie
 List<string> csrfs;//用于请求站点的token的key 以及 value
 
 public CSRFToken(string url)
 {
  //校验传输安全
  if (!string.IsNullOrWhiteSpace(url))
  {
  try
  {
   //设置请求的头信息.获取url的host
   var _http = new HttpHelper(url);
   string cookie;
   string html = _http.CreateGetHttpResponseForPC(out cookie);
   this.cookie = cookie;
 
   string headRegex = @"<meta name=""_csrf.*"" content="".*""/>";
 
   MatchCollection matches = Regex.Matches(html, headRegex);
   Regex re = new Regex("(?<=content=\").*?(?=\")", RegexOptions.None);
   csrfs = new List<string>();
   foreach (Match math in matches)
   {
 
   MatchCollection mc = re.Matches(math.Value);
   foreach (Match ma in mc)
   {
    csrfs.Add(ma.Value);
   }
   }
 
  }
  catch (Exception e)
  {
 
  }
  }
 }
 
 public String getCookie()
 {
  return cookie;
 }
 public void setCookie(String cookie)
 {
  this.cookie = cookie;
 }
 public List<string> getCsrf_token()
 {
  return csrfs;
 }
 }
httpHelper

public string CreatePostHttpResponse(IDictionary<string, string> headers, IDictionary<string, string> parameters)
  {
   HttpWebRequest request = null;
   //HTTPSQ请求 
   UTF8Encoding encoding = new System.Text.UTF8Encoding();
   ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
   request = WebRequest.Create(_baseIPAddress) as HttpWebRequest;
   request.ProtocolVersion = HttpVersion.Version10;
   ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;
   request.Method = "POST";
   request.ContentType = "application/x-www-form-urlencoded";
   // request.ContentType = "application/json";
   request.UserAgent = DefaultUserAgent;
   //request.Headers.Add("X-CSRF-TOKEN", "bc0cc533-60cc-484a-952d-0b4c1a95672c");
   //request.Referer = "https://www.asianacargo.com/tracking/viewTraceAirWaybill.do";
 
   //request.Headers.Add("Origin", "https://www.asianacargo.com");
   //request.Headers.Add("Cookie", "JSESSIONID=HP21d2Dq5FoSlG4Fyw4slWwHb0-Sl1CG6jGtj7HE41e5f4aN_R1p!-435435446!117330181");
   //request.Host = "www.asianacargo.com";
 
 
   if (!(headers == null || headers.Count == 0))
   {
 
    foreach (string key in headers.Keys)
    {
     request.Headers.Add(key, headers[key]);
    }
 
   }
 
 
   //如果需要POST数据  
   if (!(parameters == null || parameters.Count == 0))
   {
    StringBuilder buffer = new StringBuilder();
    int i = 0;
    foreach (string key in parameters.Keys)
    {
     if (i > 0)
     {
      buffer.AppendFormat("&{0}={1}", key, parameters[key]);
     }
     else
     {
      buffer.AppendFormat("{0}={1}", key, parameters[key]);
     }
     i++;
    }
    byte[] data = encoding.GetBytes(buffer.ToString());
    using (Stream stream = request.GetRequestStream())
    {
     stream.Write(data, 0, data.Length);
    }
   }
 
   HttpWebResponse response;
 
   try
   {
    //获得响应流
    response = (HttpWebResponse)request.GetResponse();
    Stream s = response.GetResponseStream();
 
    StreamReader readStream = new StreamReader(s, Encoding.UTF8);
    string SourceCode = readStream.ReadToEnd();
    response.Close();
    readStream.Close();
    return SourceCode;
   }
   catch (WebException ex)
   {
    response = ex.Response as HttpWebResponse; return null;
   }
 
  }
 
 public string CreateGetHttpResponse(out string cookie)
  {
   HttpWebRequest request = null;
   //HTTPSQ请求 
   UTF8Encoding encoding = new System.Text.UTF8Encoding();
   ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
   request = WebRequest.Create(_baseIPAddress) as HttpWebRequest;
   request.ProtocolVersion = HttpVersion.Version10;
   ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;
   request.Method = "GET";
   request.ContentType = "application/x-www-form-urlencoded";
   request.UserAgent = DefaultUserAgent;
 
   HttpWebResponse response;
 
   try
   {
    //获得响应流
    response = (HttpWebResponse)request.GetResponse();
 
    cookie = response.Headers["Set-Cookie"];
    Stream s = response.GetResponseStream();
 
    StreamReader readStream = new StreamReader(s, Encoding.UTF8);
    string SourceCode = readStream.ReadToEnd();
    response.Close();
    readStream.Close();
    return SourceCode;
   }
   catch (WebException ex)
   {
    response = ex.Response as HttpWebResponse;
    cookie = "";
    return null;
   }
 
  }

爬取程序

爬取结果

浏览器结果

注意事项与结论
1，不同的网站，获取cstf的方式不一样，无论怎么做，只要信息传到前台我们都可以有相应的方法来获取。

2，请求时候的http验证可能不一样，测试的某航空公python基础教程司物流信息的时候，http请求的安全协议是tis12。

ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11; 还有其他参数比如UserAgent后台可能也会验证

3,基于如上航空公司，发现它的cookie和cstf_token一定时间内不会改变，那么当实际爬取的时候可以考虑缓存cookie以及cstf_token,只有当请求失败的时候，才重新获取

内容总结

以上是互联网集市为您收集整理的C# 关于爬取网站数据遇到csrf-token的分析与解决全部内容，希望文章能够帮你解决C# 关于爬取网站数据遇到csrf-token的分析与解决所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。

内容备注

版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。

内容手机端