cloudflare ddos​​ portectionのページからhtmlを取得するにはどうすればよいですか？

Question

私はウェブページのデータを取得するためにhtmlagilityを使用しますが、私はddosのwww.cloudflare.com保護を使用してページですべてを試しました。リダイレクトページはhtmlagilityで扱うことができません。なぜなら、metaやjsでリダイレクトしないからです。私はC＃でシミュレーションできなかったCookieをチェックしているかどうかをチェックします。私がページを取得すると、htmlコードは着陸Cloadflareページから来ます。

Answer 1

利用WebClientページのHTMLを取得するには、
私もクッキーを処理するクラスを以下、
ちょうどコンストラクタでCookieContainerインスタンスを渡す書きました。

using System; 
using System.Collections.Generic; 
using System.Configuration; 
using System.Linq; 
using System.Net; 
using System.Text; 

namespace NitinJS 
{ 
    public class SmsWebClient : WebClient 
    { 
     public SmsWebClient(CookieContainer container, Dictionary<string, string> Headers) 
      : this(container) 
     { 
      foreach (var keyVal in Headers) 
      { 
       this.Headers[keyVal.Key] = keyVal.Value; 
      } 
     } 
     public SmsWebClient(bool flgAddContentType = true) 
      : this(new CookieContainer(), flgAddContentType) 
     { 

     } 
     public SmsWebClient(CookieContainer container, bool flgAddContentType = true) 
     { 
      this.Encoding = Encoding.UTF8; 
      System.Net.ServicePointManager.Expect100Continue = false; 
      ServicePointManager.MaxServicePointIdleTime = 2000; 
      this.container = container; 

      if (flgAddContentType) 
       this.Headers["Content-Type"] = "application/json";//"application/x-www-form-urlencoded"; 
      this.Headers["Accept"] = "application/json, text/javascript, */*; q=0.01";// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; 
      //this.Headers["Accept-Encoding"] = "gzip, deflate"; 
      this.Headers["Accept-Language"] = "en-US,en;q=0.5"; 
      this.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; rv:23.0) Gecko/20100101 Firefox/23.0"; 
      this.Headers["X-Requested-With"] = "XMLHttpRequest"; 
      //this.Headers["Connection"] = "keep-alive"; 
     } 

     private readonly CookieContainer container = new CookieContainer(); 

     protected override WebRequest GetWebRequest(Uri address) 
     { 
      WebRequest r = base.GetWebRequest(address); 
      var request = r as HttpWebRequest; 
      if (request != null) 
      { 
       request.CookieContainer = container; 
       request.Timeout = 3600000; //20 * 60 * 1000 
      } 
      return r; 
     } 

     protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result) 
     { 
      WebResponse response = base.GetWebResponse(request, result); 
      ReadCookies(response); 
      return response; 
     } 

     protected override WebResponse GetWebResponse(WebRequest request) 
     { 
      WebResponse response = base.GetWebResponse(request); 
      ReadCookies(response); 
      return response; 
     } 

     private void ReadCookies(WebResponse r) 
     { 
      var response = r as HttpWebResponse; 
      if (response != null) 
      { 
       CookieCollection cookies = response.Cookies; 
       container.Add(cookies); 
      } 
     } 
    } 
} 

USAGE：

CookieContainer cookies = new CookieContainer(); 
SmsWebClient client = new SmsWebClient(cookies); 
string html = client.DownloadString("http://www.google.com"); 

Answer 2

私はまた、いくつかの時間前に、この問題が発生しました。 real解決策は、cloudflareウェブサイトがあなたに与える挑戦を解決するでしょう（javascriptを使って正解を計算し、それを送り返し、次にあなたがウェブサイトを見ることができるクッキー/トークンを受け取る必要があります）。だから、あなたが正常になるだろう、すべてが終わりで

のようなページですが、私はちょうど実行シェルとのpythonスクリプトと呼ばれます。私はthis github forkの中で提供されるモジュールを使用しました。これは、C＃のcloudflare anti-dDoSページの迂回を実現するための出発点としても役立ちます。

私の個人的な使い方のために書いたpythonスクリプトは、ファイルにクッキーを書き込んだだけです。私は後でC＃を使用してそれを読んで、CookieJarに保存してC＃内のページを閲覧し続けます。

#!/usr/bin/env python 
import cfscrape 
import sys 

scraper = cfscrape.create_scraper() # returns a requests.Session object 
fd = open("cookie.txt", "w") 
c = cfscrape.get_cookie_string(sys.argv[1]) 
fd.write(str(c)) 
fd.close() 
print(c) 

EDIT：これを繰り返すには、これはクッキーを行うには少しだけを持っています！ Cloudflareは、javascriptコマンドを使用してREALチャレンジを解決するように強制します。クッキーを受け入れ、後で使用するのは簡単ではありません。 https://github.com/Anorov/cloudflare-scrape/blob/master/cfscrape/init.pyと〜40行のjavascriptエミュレーションを見て、課題を解決してください。

Edit2：保護を回避するために何かを書くのではなく、完全な本物のブラウザオブジェクト（ではなくのヘッドレスブラウザ）を使用している人々も見てきました。ページが読み込まれます。 WebBrowserクラスを使用して、小さなブラウザウィンドウを作成し、適切なイベントに登録します。

Edit3： 申し訳ありませんが、私は実際にこれを行うためにC＃の方法を実装しました。ヘッダはSet-Cookieセクションが含まれていますが、時々HttpResponseクラスは、クッキーをピックアップしませんので、これは、.NET用のJavaScriptエンジンJINTを使用して、https://www.nuget.org/packages/Jint

を介して利用可能クッキー処理コードは醜いです。

using System; 
using System.Net; 
using System.IO; 
using System.Text.RegularExpressions; 
using System.Web; 
using System.Collections; 
using System.Threading; 

namespace Cloudflare_Evader 
{ 
    public class CloudflareEvader 
    { 
     /// <summary> 
     /// Tries to return a webclient with the neccessary cookies installed to do requests for a cloudflare protected website. 
     /// </summary> 
     /// <param name="url">The page which is behind cloudflare's anti-dDoS protection</param> 
     /// <returns>A WebClient object or null on failure</returns> 
     public static WebClient CreateBypassedWebClient(string url) 
     { 
      var JSEngine = new Jint.Engine(); //Use this JavaScript engine to compute the result. 

      //Download the original page 
      var uri = new Uri(url); 
      HttpWebRequest req =(HttpWebRequest) WebRequest.Create(url); 
      req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0"; 
      //Try to make the usual request first. If this fails with a 503, the page is behind cloudflare. 
      try 
      { 
       var res = req.GetResponse(); 
       string html = ""; 
       using (var reader = new StreamReader(res.GetResponseStream())) 
        html = reader.ReadToEnd(); 
       return new WebClient(); 
      } 
      catch (WebException ex) //We usually get this because of a 503 service not available. 
      { 
       string html = ""; 
       using (var reader = new StreamReader(ex.Response.GetResponseStream())) 
        html = reader.ReadToEnd(); 
       //If we get on the landing page, Cloudflare gives us a User-ID token with the cookie. We need to save that and use it in the next request. 
       var cookie_container = new CookieContainer(); 
       //using a custom function because ex.Response.Cookies returns an empty set ALTHOUGH cookies were sent back. 
       var initial_cookies = GetAllCookiesFromHeader(ex.Response.Headers["Set-Cookie"], uri.Host); 
       foreach (Cookie init_cookie in initial_cookies) 
        cookie_container.Add(init_cookie); 

       /* solve the actual challenge with a bunch of RegEx's. Copy-Pasted from the python scrapper version.*/ 
       var challenge = Regex.Match(html, "name=\"jschl_vc\" value=\"(\\w+)\"").Groups[1].Value; 
       var challenge_pass = Regex.Match(html, "name=\"pass\" value=\"(.+?)\"").Groups[1].Value; 

       var builder = Regex.Match(html, @"setTimeout\(function\(\){\s+(var t,r,a,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n").Groups[1].Value; 
       builder = Regex.Replace(builder, @"a\.value =(.+?) \+ .+?;", "$1"); 
       builder = Regex.Replace(builder, @"\s{3,}[a-z](?: = |\.).+", ""); 

       //Format the javascript.. 
       builder = Regex.Replace(builder, @"[\n\\']", ""); 

       //Execute it. 
       long solved = long.Parse(JSEngine.Execute(builder).GetCompletionValue().ToObject().ToString()); 
       solved += uri.Host.Length; //add the length of the domain to it. 

       Console.WriteLine("***** SOLVED CHALLENGE ******: " + solved); 
       Thread.Sleep(3000); //This sleeping IS requiered or cloudflare will not give you the token!! 

       //Retreive the cookies. Prepare the URL for cookie exfiltration. 
       string cookie_url = string.Format("{0}://{1}/cdn-cgi/l/chk_jschl", uri.Scheme, uri.Host); 
       var uri_builder = new UriBuilder(cookie_url); 
       var query = HttpUtility.ParseQueryString(uri_builder.Query); 
       //Add our answers to the GET query 
       query["jschl_vc"] = challenge; 
       query["jschl_answer"] = solved.ToString(); 
       query["pass"] = challenge_pass; 
       uri_builder.Query = query.ToString(); 

       //Create the actual request to get the security clearance cookie 
       HttpWebRequest cookie_req = (HttpWebRequest) WebRequest.Create(uri_builder.Uri); 
       cookie_req.AllowAutoRedirect = false; 
       cookie_req.CookieContainer = cookie_container; 
       cookie_req.Referer = url; 
       cookie_req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0"; 
       //We assume that this request goes through well, so no try-catch 
       var cookie_resp = (HttpWebResponse)cookie_req.GetResponse(); 
       //The response *should* contain the security clearance cookie! 
       if (cookie_resp.Cookies.Count != 0) //first check if the HttpWebResponse has picked up the cookie. 
        foreach (Cookie cookie in cookie_resp.Cookies) 
         cookie_container.Add(cookie); 
       else //otherwise, use the custom function again 
       { 
        //the cookie we *hopefully* received here is the cloudflare security clearance token. 
        if (cookie_resp.Headers["Set-Cookie"] != null) 
        { 
         var cookies_parsed = GetAllCookiesFromHeader(cookie_resp.Headers["Set-Cookie"], uri.Host); 
         foreach (Cookie cookie in cookies_parsed) 
          cookie_container.Add(cookie); 
        } 
        else 
        { 
         //No security clearence? something went wrong.. return null. 
         //Console.WriteLine("MASSIVE ERROR: COULDN'T GET CLOUDFLARE CLEARANCE!"); 
         return null; 
        } 
       } 
       //Create a custom webclient with the two cookies we already acquired. 
       WebClient modedWebClient = new WebClientEx(cookie_container); 
       modedWebClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0"); 
       modedWebClient.Headers.Add("Referer", url); 
       return modedWebClient; 
      } 
     } 

     /* Credit goes to https://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect 
      (user https://stackoverflow.com/users/541404/cameron-tinker) for these functions 
     */ 
     public static CookieCollection GetAllCookiesFromHeader(string strHeader, string strHost) 
     { 
      ArrayList al = new ArrayList(); 
      CookieCollection cc = new CookieCollection(); 
      if (strHeader != string.Empty) 
      { 
       al = ConvertCookieHeaderToArrayList(strHeader); 
       cc = ConvertCookieArraysToCookieCollection(al, strHost); 
      } 
      return cc; 
     } 

     private static ArrayList ConvertCookieHeaderToArrayList(string strCookHeader) 
     { 
      strCookHeader = strCookHeader.Replace("\r", ""); 
      strCookHeader = strCookHeader.Replace("\n", ""); 
      string[] strCookTemp = strCookHeader.Split(','); 
      ArrayList al = new ArrayList(); 
      int i = 0; 
      int n = strCookTemp.Length; 
      while (i < n) 
      { 
       if (strCookTemp[i].IndexOf("expires=", StringComparison.OrdinalIgnoreCase) > 0) 
       { 
        al.Add(strCookTemp[i] + "," + strCookTemp[i + 1]); 
        i = i + 1; 
       } 
       else 
        al.Add(strCookTemp[i]); 
       i = i + 1; 
      } 
      return al; 
     } 

     private static CookieCollection ConvertCookieArraysToCookieCollection(ArrayList al, string strHost) 
     { 
      CookieCollection cc = new CookieCollection(); 

      int alcount = al.Count; 
      string strEachCook; 
      string[] strEachCookParts; 
      for (int i = 0; i < alcount; i++) 
      { 
       strEachCook = al[i].ToString(); 
       strEachCookParts = strEachCook.Split(';'); 
       int intEachCookPartsCount = strEachCookParts.Length; 
       string strCNameAndCValue = string.Empty; 
       string strPNameAndPValue = string.Empty; 
       string strDNameAndDValue = string.Empty; 
       string[] NameValuePairTemp; 
       Cookie cookTemp = new Cookie(); 

       for (int j = 0; j < intEachCookPartsCount; j++) 
       { 
        if (j == 0) 
        { 
         strCNameAndCValue = strEachCookParts[j]; 
         if (strCNameAndCValue != string.Empty) 
         { 
          int firstEqual = strCNameAndCValue.IndexOf("="); 
          string firstName = strCNameAndCValue.Substring(0, firstEqual); 
          string allValue = strCNameAndCValue.Substring(firstEqual + 1, strCNameAndCValue.Length - (firstEqual + 1)); 
          cookTemp.Name = firstName; 
          cookTemp.Value = allValue; 
         } 
         continue; 
        } 
        if (strEachCookParts[j].IndexOf("path", StringComparison.OrdinalIgnoreCase) >= 0) 
        { 
         strPNameAndPValue = strEachCookParts[j]; 
         if (strPNameAndPValue != string.Empty) 
         { 
          NameValuePairTemp = strPNameAndPValue.Split('='); 
          if (NameValuePairTemp[1] != string.Empty) 
           cookTemp.Path = NameValuePairTemp[1]; 
          else 
           cookTemp.Path = "/"; 
         } 
         continue; 
        } 

        if (strEachCookParts[j].IndexOf("domain", StringComparison.OrdinalIgnoreCase) >= 0) 
        { 
         strPNameAndPValue = strEachCookParts[j]; 
         if (strPNameAndPValue != string.Empty) 
         { 
          NameValuePairTemp = strPNameAndPValue.Split('='); 

          if (NameValuePairTemp[1] != string.Empty) 
           cookTemp.Domain = NameValuePairTemp[1]; 
          else 
           cookTemp.Domain = strHost; 
         } 
         continue; 
        } 
       } 

       if (cookTemp.Path == string.Empty) 
        cookTemp.Path = "/"; 
       if (cookTemp.Domain == string.Empty) 
        cookTemp.Domain = strHost; 
       cc.Add(cookTemp); 
      } 
      return cc; 
     } 
    } 

    /*Credit goes to https://stackoverflow.com/questions/1777221/using-cookiecontainer-with-webclient-class 
(user https://stackoverflow.com/users/129124/pavel-savara) */ 
    public class WebClientEx : WebClient 
    { 
     public WebClientEx(CookieContainer container) 
     { 
      this.container = container; 
     } 

     public CookieContainer CookieContainer 
     { 
      get { return container; } 
      set { container = value; } 
     } 

     private CookieContainer container = new CookieContainer(); 

     protected override WebRequest GetWebRequest(Uri address) 
     { 
      WebRequest r = base.GetWebRequest(address); 
      var request = r as HttpWebRequest; 
      if (request != null) 
      { 
       request.CookieContainer = container; 
      } 
      return r; 
     } 

     protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result) 
     { 
      WebResponse response = base.GetWebResponse(request, result); 
      ReadCookies(response); 
      return response; 
     } 

     protected override WebResponse GetWebResponse(WebRequest request) 
     { 
      WebResponse response = base.GetWebResponse(request); 
      ReadCookies(response); 
      return response; 
     } 

     private void ReadCookies(WebResponse r) 
     { 
      var response = r as HttpWebResponse; 
      if (response != null) 
      { 
       CookieCollection cookies = response.Cookies; 
       container.Add(cookies); 
      } 
     } 
    } 
} 

この機能は、解決されたチャレンジとクッキーが内部にあるWebクライアントを返します。次のように使用できます。

static void Main(string[] args) 
{ 
    WebClient client = null; 
    while (client == null) 
    { 
     Console.WriteLine("Trying.."); 
     client = CloudflareEvader.CreateBypassedWebClient("http://anilinkz.tv"); 
    } 
    Console.WriteLine("Solved! We're clear to go"); 
     Console.WriteLine(client.DownloadString("http://anilinkz.tv/anime-list")); 

    Console.ReadLine(); 
}