fix cloudscraper (non proprio perfetto)

core/httptools.py

@@ -438,7 +438,7 @@ def downloadpage(url, **opt):
         opt['proxy_retries_counter'] += 1
 
         session = cloudscraper.create_scraper()
-        session.verify = False
+        # session.verify = False
         if opt.get('cookies', True):
             session.cookies = cj
         session.headers.update(req_headers)

lib/cloudscraper/__init__.py

@@ -4,13 +4,17 @@ import sys
 import ssl
 import requests
 
+try:
+    import copyreg
+except ImportError:
+    import copy_reg as copyreg
+
 from copy import deepcopy
 from time import sleep
 from collections import OrderedDict
 
 from requests.sessions import Session
 from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util.ssl_ import create_urllib3_context
 
 from .interpreters import JavaScriptInterpreter
 from .reCaptcha import reCaptcha
@@ -31,153 +35,210 @@ try:
 except ImportError:
     from urllib.parse import urlparse
 
-##########################################################################################################################################################
+# ------------------------------------------------------------------------------- #
 
-__version__ = '1.1.24'
+__version__ = '1.2.14'
 
-BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.'
+# ------------------------------------------------------------------------------- #
-
-##########################################################################################################################################################
 
 
-# class CipherSuiteAdapter(HTTPAdapter):
-#
-#     def __init__(self, cipherSuite=None, **kwargs):
-#         self.cipherSuite = cipherSuite
-#
-#         self.ssl_context = create_urllib3_context(
-#             ssl_version=ssl.PROTOCOL_TLS,
-#             ciphers=self.cipherSuite
-#         )
-#
-#         super(CipherSuiteAdapter, self).__init__(**kwargs)
 class CipherSuiteAdapter(HTTPAdapter):
 
-    def __init__(self, cipherSuite=None, **kwargs):
+    __attrs__ = [
-        self.cipherSuite = cipherSuite
+        'ssl_context',
+        'max_retries',
+        'config',
+        '_pool_connections',
+        '_pool_maxsize',
+        '_pool_block'
+    ]
 
-        if hasattr(ssl, 'PROTOCOL_TLS'):
+    def __init__(self, *args, **kwargs):
-            self.ssl_context = create_urllib3_context(
+        self.ssl_context = kwargs.pop('ssl_context', None)
-                ssl_version=getattr(ssl, 'PROTOCOL_TLSv1_3', ssl.PROTOCOL_TLSv1_2),
+        self.cipherSuite = kwargs.pop('cipherSuite', None)
-                ciphers=self.cipherSuite
+
-            )
+        if not self.ssl_context:
-        else:
+            self.ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
-            self.ssl_context = create_urllib3_context(ssl_version=ssl.PROTOCOL_TLSv1)
+            self.ssl_context.set_ciphers(self.cipherSuite)
+            self.ssl_context.options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1)
 
         super(CipherSuiteAdapter, self).__init__(**kwargs)
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
 
     def init_poolmanager(self, *args, **kwargs):
         kwargs['ssl_context'] = self.ssl_context
         return super(CipherSuiteAdapter, self).init_poolmanager(*args, **kwargs)
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
 
     def proxy_manager_for(self, *args, **kwargs):
         kwargs['ssl_context'] = self.ssl_context
         return super(CipherSuiteAdapter, self).proxy_manager_for(*args, **kwargs)
 
-##########################################################################################################################################################
+# ------------------------------------------------------------------------------- #
 
 
 class CloudScraper(Session):
+
     def __init__(self, *args, **kwargs):
-        self.allow_brotli = kwargs.pop('allow_brotli', True if 'brotli' in sys.modules.keys() else False)
         self.debug = kwargs.pop('debug', False)
         self.delay = kwargs.pop('delay', None)
+        self.cipherSuite = kwargs.pop('cipherSuite', None)
         self.interpreter = kwargs.pop('interpreter', 'js2py')
         self.recaptcha = kwargs.pop('recaptcha', {})
+        self.allow_brotli = kwargs.pop(
+            'allow_brotli',
+            True if 'brotli' in sys.modules.keys() else False
+        )
+        self.user_agent = User_Agent(
+            allow_brotli=self.allow_brotli,
+            browser=kwargs.pop('browser', None)
+        )
 
-        self.cipherSuite = None
+        self._solveDepthCnt = 0
+        self.solveDepth = kwargs.pop('solveDepth', 3)
 
         super(CloudScraper, self).__init__(*args, **kwargs)
 
+        # pylint: disable=E0203
         if 'requests' in self.headers['User-Agent']:
+            # ------------------------------------------------------------------------------- #
             # Set a random User-Agent if no custom User-Agent has been set
-            self.headers = User_Agent(allow_brotli=self.allow_brotli).headers
+            # ------------------------------------------------------------------------------- #
+            self.headers = self.user_agent.headers
 
-        self.mount('https://', CipherSuiteAdapter(self.loadCipherSuite()))
+        self.mount(
+            'https://',
+            CipherSuiteAdapter(
+                cipherSuite=self.loadCipherSuite() if not self.cipherSuite else self.cipherSuite
+            )
+        )
 
-    ##########################################################################################################################################################
+        # purely to allow us to pickle dump
+        copyreg.pickle(ssl.SSLContext, lambda obj: (obj.__class__, (obj.protocol,)))
+
+    # ------------------------------------------------------------------------------- #
+    # Allow us to pickle our session back with all variables
+    # ------------------------------------------------------------------------------- #
+
+    def __getstate__(self):
+        return self.__dict__
+
+    # ------------------------------------------------------------------------------- #
+    # debug the request via the response
+    # ------------------------------------------------------------------------------- #
 
     @staticmethod
     def debugRequest(req):
         try:
             print(dump.dump_all(req).decode('utf-8'))
-        except:  # noqa
+        except ValueError as e:
-            pass
+            print("Debug Error: {}".format(getattr(e, 'message', e)))
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
+    # Decode Brotli on older versions of urllib3 manually
+    # ------------------------------------------------------------------------------- #
+
+    def decodeBrotli(self, resp):
+        if requests.packages.urllib3.__version__ < '1.25.1' and resp.headers.get('Content-Encoding') == 'br':
+            if self.allow_brotli and resp._content:
+                resp._content = brotli.decompress(resp.content)
+            else:
+                logging.warning(
+                    'You\'re running urllib3 {}, Brotli content detected, '
+                    'Which requires manual decompression, '
+                    'But option allow_brotli is set to False, '
+                    'We will not continue to decompress.'.format(requests.packages.urllib3.__version__)
+                )
+
+        return resp
+
+    # ------------------------------------------------------------------------------- #
+    # construct a cipher suite of ciphers the system actually supports
+    # ------------------------------------------------------------------------------- #
 
     def loadCipherSuite(self):
         if self.cipherSuite:
             return self.cipherSuite
 
-        self.cipherSuite = ''
+        if hasattr(ssl, 'Purpose') and hasattr(ssl.Purpose, 'SERVER_AUTH'):
-
+            for cipher in self.user_agent.cipherSuite[:]:
-        if hasattr(ssl, 'PROTOCOL_TLS'):
-            ciphers = [
-                'TLS13-AES-128-GCM-SHA256',
-                'TLS13-AES-256-GCM-SHA384',
-                'TLS13-CHACHA20-POLY1305-SHA256',
-                'ECDHE-ECDSA-CHACHA20-POLY1305',
-                'ECDHE-ECDSA-AES128-GCM-SHA256',
-                'ECDHE-ECDSA-AES128-SHA',
-                'ECDHE-ECDSA-AES128-SHA256',
-                'ECDHE-ECDSA-AES256-GCM-SHA384',
-                'ECDHE-ECDSA-AES256-SHA',
-                'ECDHE-ECDSA-AES256-SHA384',
-                # Slip in some additional intermediate compatibility ciphers, This should help out users for non Cloudflare based sites.
-                'ECDHE-RSA-AES128-SHA256',
-                'ECDHE-RSA-AES256-SHA384',
-                'ECDHE-RSA-AES256-GCM-SHA384',
-                'DHE-RSA-AES128-GCM-SHA256',
-                'DHE-RSA-AES256-GCM-SHA384'
-            ]
-
-            ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
-
-            for cipher in ciphers:
                 try:
-                    ctx.set_ciphers(cipher)
+                    context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
-                    self.cipherSuite = '{}:{}'.format(self.cipherSuite, cipher).rstrip(':').lstrip(':')
+                    context.set_ciphers(cipher)
-                except ssl.SSLError:
+                except:
-                    pass
+                    self.user_agent.cipherSuite.remove(cipher)
 
+            if self.user_agent.cipherSuite:
+                self.cipherSuite = ':'.join(self.user_agent.cipherSuite)
                 return self.cipherSuite
 
-    ##########################################################################################################################################################
+        sys.tracebacklimit = 0
+        raise RuntimeError("The OpenSSL on this system does not meet the minimum cipher requirements.")
+
+    # ------------------------------------------------------------------------------- #
+    # Our hijacker request function
+    # ------------------------------------------------------------------------------- #
 
     def request(self, method, url, *args, **kwargs):
-        ourSuper = super(CloudScraper, self)
+        # pylint: disable=E0203
-        resp = ourSuper.request(method, url, *args, **kwargs)
+        if kwargs.get('proxies') and kwargs.get('proxies') != self.proxies:
+            self.proxies = kwargs.get('proxies')
 
-        if requests.packages.urllib3.__version__ < '1.25.1' and resp.headers.get('Content-Encoding') == 'br':
+        resp = self.decodeBrotli(
-            if self.allow_brotli and resp._content:
+            super(CloudScraper, self).request(method, url, *args, **kwargs)
-                resp._content = brotli.decompress(resp.content)
+        )
-            else:
-                logging.warning('Brotli content detected, But option is disabled, we will not continue.')
-                return resp
 
+        # ------------------------------------------------------------------------------- #
         # Debug request
+        # ------------------------------------------------------------------------------- #
+
         if self.debug:
             self.debugRequest(resp)
 
         # Check if Cloudflare anti-bot is on
-        if self.isChallengeRequest(resp):
+        if self.is_Challenge_Request(resp):
-            if resp.request.method != 'GET':
+            # ------------------------------------------------------------------------------- #
-                # Work around if the initial request is not a GET,
+            # Try to solve the challenge and send it back
-                # Supersede with a GET then re-request the original METHOD.
+            # ------------------------------------------------------------------------------- #
-                self.request('GET', resp.url)
+
-                resp = ourSuper.request(method, url, *args, **kwargs)
+            if self._solveDepthCnt >= self.solveDepth:
+                sys.tracebacklimit = 0
+                _ = self._solveDepthCnt
+                self._solveDepthCnt = 0
+                raise RuntimeError("!!Loop Protection!! We have tried to solve {} time(s) in a row.".format(_))
+
+            self._solveDepthCnt += 1
+
+            resp = self.Challenge_Response(resp, **kwargs)
         else:
-                # Solve Challenge
+            if resp.status_code not in [302, 429, 503]:
-                resp = self.sendChallengeResponse(resp, **kwargs)
+                self._solveDepthCnt = 0
 
         return resp
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
+    # check if the response contains a valid Cloudflare challenge
+    # ------------------------------------------------------------------------------- #
+
+    @staticmethod
+    def is_IUAM_Challenge(resp):
+        try:
+            return (
+                resp.headers.get('Server', '').startswith('cloudflare')
+                and resp.status_code in [429, 503]
+                and re.search(
+                    r'action="/.*?__cf_chl_jschl_tk__=\S+".*?name="jschl_vc"\svalue=.*?',
+                    resp.text,
+                    re.M | re.DOTALL
+                )
+            )
+        except AttributeError:
+            pass
+
+        return False
+
     # ------------------------------------------------------------------------------- #
     # check if the response contains a valid Cloudflare reCaptcha challenge
     # ------------------------------------------------------------------------------- #
@@ -199,41 +260,40 @@ class CloudScraper(Session):
 
         return False
 
-    @staticmethod
+    # ------------------------------------------------------------------------------- #
-    def isChallengeRequest(resp):
+    # Wrapper for is_reCaptcha_Challenge and is_IUAM_Challenge
-        if resp.headers.get('Server', '').startswith('cloudflare'):
+    # ------------------------------------------------------------------------------- #
-            return (
+
-                resp.status_code in [403, 429, 503]
+    def is_Challenge_Request(self, resp):
-                and (
+        if self.is_reCaptcha_Challenge(resp) or self.is_IUAM_Challenge(resp):
-                    all(s in resp.content for s in [b'jschl_vc', b'jschl_answer'])
+            return True
-                    or
-                    all(s in resp.content for s in [b'why_captcha', b'/cdn-cgi/l/chk_captcha'])
-                )
-            )
 
         return False
 
-
     # ------------------------------------------------------------------------------- #
     # Try to solve cloudflare javascript challenge.
     # ------------------------------------------------------------------------------- #
 
     @staticmethod
-    def IUAM_Challenge_Response(body, domain, interpreter):
+    def IUAM_Challenge_Response(body, url, interpreter):
         try:
             challengeUUID = re.search(
-                r'__cf_chl_jschl_tk__=(?P<challengeUUID>\S+)"',
+                r'id="challenge-form" action="(?P<challengeUUID>\S+)"',
                 body, re.M | re.DOTALL
-            ).groupdict().get('challengeUUID')
+            ).groupdict().get('challengeUUID', '')
-            params = OrderedDict(re.findall(r'name="(r|jschl_vc|pass)"\svalue="(.*?)"', body))
+            payload = OrderedDict(re.findall(r'name="(r|jschl_vc|pass)"\svalue="(.*?)"', body))
         except AttributeError:
             sys.tracebacklimit = 0
-            raise RuntimeError("Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly.")
+            raise RuntimeError(
+                "Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly."
+            )
+
+        hostParsed = urlparse(url)
 
         try:
-            params['jschl_answer'] = JavaScriptInterpreter.dynamicImport(
+            payload['jschl_answer'] = JavaScriptInterpreter.dynamicImport(
                 interpreter
-            ).solveChallenge(body, domain)
+            ).solveChallenge(body, hostParsed.netloc)
         except Exception as e:
             raise RuntimeError(
                 'Unable to parse Cloudflare anti-bots page: {}'.format(
@@ -242,16 +302,23 @@ class CloudScraper(Session):
             )
 
         return {
-            'url': 'https://{}/'.format(domain),
+            'url': '{}://{}{}'.format(
-            'params': {'__cf_chl_jschl_tk__': challengeUUID},
+                hostParsed.scheme,
-            'data': params
+                hostParsed.netloc,
+                challengeUUID
+            ),
+            'data': payload
         }
 
+    # ------------------------------------------------------------------------------- #
+    #  Try to solve the reCaptcha challenge via 3rd party.
+    # ------------------------------------------------------------------------------- #
+
     @staticmethod
     def reCaptcha_Challenge_Response(provider, provider_params, body, url):
         try:
-            params = re.search(
+            payload = re.search(
-                r'(name="r"\svalue="(?P<r>\S+)"|).*?__cf_chl_captcha_tk__=(?P<challengeUUID>\S+)".*?'
+                r'(name="r"\svalue="(?P<r>\S+)"|).*?challenge-form" action="(?P<challengeUUID>\S+)".*?'
                 r'data-ray="(?P<data_ray>\S+)".*?data-sitekey="(?P<site_key>\S+)"',
                 body, re.M | re.DOTALL
             ).groupdict()
@@ -261,24 +328,30 @@ class CloudScraper(Session):
                 "Cloudflare reCaptcha detected, unfortunately we can't extract the parameters correctly."
             )
 
+        hostParsed = urlparse(url)
         return {
-            'url': url,
+            'url': '{}://{}{}'.format(
-            'params': {'__cf_chl_captcha_tk__': params.get('challengeUUID')},
+                hostParsed.scheme,
+                hostParsed.netloc,
+                payload.get('challengeUUID', '')
+            ),
             'data': OrderedDict([
-                ('r', ''),
+                ('r', payload.get('r', '')),
-                ('id', params.get('data_ray')),
+                ('id', payload.get('data_ray')),
                 (
                     'g-recaptcha-response',
                     reCaptcha.dynamicImport(
                         provider.lower()
-                    ).solveCaptcha(url, params.get('site_key'), provider_params)
+                    ).solveCaptcha(url, payload.get('site_key'), provider_params)
                 )
             ])
         }
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
+    # Attempt to handle and send the challenge response back to cloudflare
+    # ------------------------------------------------------------------------------- #
 
-    def sendChallengeResponse(self, resp, **original_kwargs):
+    def Challenge_Response(self, resp, **kwargs):
         if self.is_reCaptcha_Challenge(resp):
             # ------------------------------------------------------------------------------- #
             # double down on the request as some websites are only checking
@@ -342,7 +415,7 @@ class CloudScraper(Session):
 
             submit_url = self.IUAM_Challenge_Response(
                 resp.text,
-                urlparse(resp.url).netloc,
+                resp.url,
                 self.interpreter
             )
 
@@ -362,31 +435,32 @@ class CloudScraper(Session):
 
             cloudflare_kwargs = deepcopy(kwargs)
             cloudflare_kwargs['allow_redirects'] = False
-            cloudflare_kwargs['params'] = updateAttr(cloudflare_kwargs, 'params', submit_url['params'])
+            cloudflare_kwargs['data'] = updateAttr(
-            cloudflare_kwargs['data'] = updateAttr(cloudflare_kwargs, 'data', submit_url['data'])
+                cloudflare_kwargs,
-            cloudflare_kwargs['headers'] = updateAttr(cloudflare_kwargs, 'headers', {'Referer': resp.url})
+                'data',
+                submit_url['data']
+            )
+            cloudflare_kwargs['headers'] = updateAttr(
+                cloudflare_kwargs,
+                'headers',
+                {
+                    'Referer': resp.url
+                }
+            )
 
-            self.request(
+            return self.request(
                 'POST',
                 submit_url['url'],
                 **cloudflare_kwargs
             )
 
         # ------------------------------------------------------------------------------- #
-        # Request the original query request and return it
+        # We shouldn't be here.... Re-request the original query and process again....
-        # ------------------------------------------------------------------------------- #
-
-        return self.request(resp.request.method, resp.url, **kwargs)
-
-        # ------------------------------------------------------------------------------- #
-        # Request the original query request and return it
         # ------------------------------------------------------------------------------- #
 
         return self.request(resp.request.method, resp.url, **kwargs)
     # ------------------------------------------------------------------------------- #
 
-    ##########################################################################################################################################################
-
     @classmethod
     def create_scraper(cls, sess=None, **kwargs):
         """
@@ -395,25 +469,30 @@ class CloudScraper(Session):
         scraper = cls(**kwargs)
 
         if sess:
-            attrs = ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']
+            for attr in ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']:
-            for attr in attrs:
                 val = getattr(sess, attr, None)
                 if val:
                     setattr(scraper, attr, val)
 
         return scraper
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
-
     # Functions for integrating cloudscraper with other applications and scripts
+    # ------------------------------------------------------------------------------- #
+
     @classmethod
     def get_tokens(cls, url, **kwargs):
         scraper = cls.create_scraper(
-            debug=kwargs.pop('debug', False),
+            **{
-            delay=kwargs.pop('delay', None),
+                field: kwargs.pop(field, None) for field in [
-            interpreter=kwargs.pop('interpreter', 'js2py'),
+                    'allow_brotli',
-            allow_brotli=kwargs.pop('allow_brotli', True),
+                    'browser',
-            recaptcha=kwargs.pop('recaptcha', {})
+                    'debug',
+                    'delay',
+                    'interpreter',
+                    'recaptcha'
+                ] if field in kwargs
+            }
         )
 
         try:
@@ -432,7 +511,11 @@ class CloudScraper(Session):
                 cookie_domain = d
                 break
         else:
-            raise ValueError('Unable to find Cloudflare cookies. Does the site actually have Cloudflare IUAM ("I\'m Under Attack Mode") enabled?')
+            sys.tracebacklimit = 0
+            raise RuntimeError(
+                "Unable to find Cloudflare cookies. Does the site actually "
+                "have Cloudflare IUAM (I'm Under Attack Mode) enabled?"
+            )
 
         return (
             {
@@ -442,7 +525,7 @@ class CloudScraper(Session):
             scraper.headers['User-Agent']
         )
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
 
     @classmethod
     def get_cookie_string(cls, url, **kwargs):
@@ -453,7 +536,7 @@ class CloudScraper(Session):
         return '; '.join('='.join(pair) for pair in tokens.items()), user_agent
 
 
-##########################################################################################################################################################
+# ------------------------------------------------------------------------------- #
 
 create_scraper = CloudScraper.create_scraper
 get_tokens = CloudScraper.get_tokens

lib/cloudscraper/interpreters/js2py.py

@@ -15,12 +15,9 @@ class ChallengeInterpreter(JavaScriptInterpreter):
         super(ChallengeInterpreter, self).__init__('js2py')
 
     def eval(self, jsEnv, js):
-        ### blocca lo script
+        if js2py.eval_js('(+(+!+[]+[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+[!+[]+!+[]]+[+[]])+[])[+!+[]]') == '1':
-
+            logging.warning('WARNING - Please upgrade your js2py https://github.com/PiotrDabkowski/Js2Py, applying work around for the meantime.')
-        # from core.support import dbg; dbg()
+            js = jsunfuck(js)
-        # if js2py.eval_js('(+(+!+[]+[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+[!+[]+!+[]]+[+[]])+[])[+!+[]]') == '1':
-        #     logging.warning('WARNING - Please upgrade your js2py https://github.com/PiotrDabkowski/Js2Py, applying work around for the meantime.')
-        #     js = jsunfuck(js)
 
         def atob(s):
             return base64.b64decode('{}'.format(s)).decode('utf-8')

lib/cloudscraper/interpreters/nodejs.py

@@ -1,14 +1,10 @@
 import base64
-import logging
 import subprocess
+import sys
 
 from . import JavaScriptInterpreter
 
-##########################################################################################################################################################
+# ------------------------------------------------------------------------------- #
-
-BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.'
-
-##########################################################################################################################################################
 
 
 class ChallengeInterpreter(JavaScriptInterpreter):
@@ -37,10 +33,10 @@ class ChallengeInterpreter(JavaScriptInterpreter):
                 )
             raise
         except Exception:
-            logging.error('Error executing Cloudflare IUAM Javascript. %s' % BUG_REPORT)
+            sys.tracebacklimit = 0
-            raise
+            raise RuntimeError('Error executing Cloudflare IUAM Javascript in nodejs')
 
-        pass
 
+# ------------------------------------------------------------------------------- #
 
 ChallengeInterpreter()

lib/cloudscraper/interpreters/v8.py

@@ -10,17 +10,23 @@ except ImportError:
 
 from . import JavaScriptInterpreter
 
+# ------------------------------------------------------------------------------- #
+
 
 class ChallengeInterpreter(JavaScriptInterpreter):
 
     def __init__(self):
         super(ChallengeInterpreter, self).__init__('v8')
 
+    # ------------------------------------------------------------------------------- #
+
     def eval(self, jsEnv, js):
         try:
             return v8eval.V8().eval('{}{}'.format(jsEnv, js))
-        except:  # noqa
+        except (TypeError, v8eval.V8Error):
             RuntimeError('We encountered an error running the V8 Engine.')
 
 
+# ------------------------------------------------------------------------------- #
+
 ChallengeInterpreter()

lib/cloudscraper/reCaptcha/2captcha.py

@@ -0,0 +1,206 @@
+from __future__ import absolute_import
+
+import requests
+
+try:
+    import polling
+except ImportError:
+    import sys
+    sys.tracebacklimit = 0
+    raise RuntimeError("Please install the python module 'polling' via pip or download it from https://github.com/justiniso/polling/")
+
+from . import reCaptcha
+
+
+class captchaSolver(reCaptcha):
+
+    def __init__(self):
+        super(captchaSolver, self).__init__('2captcha')
+        self.host = 'https://2captcha.com'
+        self.session = requests.Session()
+
+    # ------------------------------------------------------------------------------- #
+
+    @staticmethod
+    def checkErrorStatus(response, request_type):
+        if response.status_code in [500, 502]:
+            raise RuntimeError('2Captcha: Server Side Error {}'.format(response.status_code))
+
+        errors = {
+            'in.php': {
+                "ERROR_WRONG_USER_KEY": "You've provided api_key parameter value is in incorrect format, it should contain 32 symbols.",
+                "ERROR_KEY_DOES_NOT_EXIST": "The api_key you've provided does not exists.",
+                "ERROR_ZERO_BALANCE": "You don't have sufficient funds on your account.",
+                "ERROR_PAGEURL": "pageurl parameter is missing in your request.",
+                "ERROR_NO_SLOT_AVAILABLE":
+                    "No Slots Available.\nYou can receive this error in two cases:\n"
+                    "1. If you solve ReCaptcha: the queue of your captchas that are not distributed to workers is too long. "
+                    "Queue limit changes dynamically and depends on total amount of captchas awaiting solution and usually it's between 50 and 100 captchas.\n"
+                    "2. If you solve Normal Captcha: your maximum rate for normal captchas is lower than current rate on the server."
+                    "You can change your maximum rate in your account's settings.",
+                "ERROR_IP_NOT_ALLOWED": "The request is sent from the IP that is not on the list of your allowed IPs.",
+                "IP_BANNED": "Your IP address is banned due to many frequent attempts to access the server using wrong authorization keys.",
+                "ERROR_BAD_TOKEN_OR_PAGEURL":
+                    "You can get this error code when sending ReCaptcha V2. "
+                    "That happens if your request contains invalid pair of googlekey and pageurl. "
+                    "The common reason for that is that ReCaptcha is loaded inside an iframe hosted on another domain/subdomain.",
+                "ERROR_GOOGLEKEY":
+                    "You can get this error code when sending ReCaptcha V2. "
+                    "That means that sitekey value provided in your request is incorrect: it's blank or malformed.",
+                "MAX_USER_TURN": "You made more than 60 requests within 3 seconds.Your account is banned for 10 seconds. Ban will be lifted automatically."
+            },
+            'res.php': {
+                "ERROR_CAPTCHA_UNSOLVABLE":
+                    "We are unable to solve your captcha - three of our workers were unable solve it "
+                    "or we didn't get an answer within 90 seconds (300 seconds for ReCaptcha V2). "
+                    "We will not charge you for that request.",
+                "ERROR_WRONG_USER_KEY": "You've provided api_key parameter value in incorrect format, it should contain 32 symbols.",
+                "ERROR_KEY_DOES_NOT_EXIST": "The api_key you've provided does not exists.",
+                "ERROR_WRONG_ID_FORMAT": "You've provided captcha ID in wrong format. The ID can contain numbers only.",
+                "ERROR_WRONG_CAPTCHA_ID": "You've provided incorrect captcha ID.",
+                "ERROR_BAD_DUPLICATES":
+                    "Error is returned when 100% accuracy feature is enabled. "
+                    "The error means that max numbers of tries is reached but min number of matches not found.",
+                "REPORT_NOT_RECORDED": "Error is returned to your complain request if you already complained lots of correctly solved captchas.",
+                "ERROR_IP_ADDRES":
+                    "You can receive this error code when registering a pingback (callback) IP or domain."
+                    "That happes if your request is coming from an IP address that doesn't match the IP address of your pingback IP or domain.",
+                "ERROR_TOKEN_EXPIRED": "You can receive this error code when sending GeeTest. That error means that challenge value you provided is expired.",
+                "ERROR_EMPTY_ACTION": "Action parameter is missing or no value is provided for action parameter."
+            }
+        }
+
+        if response.json().get('status') is False and response.json().get('request') in errors.get(request_type):
+            raise RuntimeError('{} {}'.format(response.json().get('request'), errors.get(request_type).get(response.json().get('request'))))
+
+    # ------------------------------------------------------------------------------- #
+
+    def reportJob(self, jobID):
+        if not jobID:
+            raise RuntimeError("2Captcha: Error bad job id to request reCaptcha.")
+
+        def _checkRequest(response):
+            if response.status_code in [200, 303] and response.json().get('status') == 1:
+                return response
+
+            self.checkErrorStatus(response, 'res.php')
+
+            return None
+
+        response = polling.poll(
+            lambda: self.session.get(
+                '{}/res.php'.format(self.host),
+                params={
+                    'key': self.api_key,
+                    'action': 'reportbad',
+                    'id': jobID,
+                    'json': '1'
+                }
+            ),
+            check_success=_checkRequest,
+            step=5,
+            timeout=180
+        )
+
+        if response:
+            return True
+        else:
+            raise RuntimeError("2Captcha: Error - Failed to report bad reCaptcha solve.")
+
+    # ------------------------------------------------------------------------------- #
+
+    def requestJob(self, jobID):
+        if not jobID:
+            raise RuntimeError("2Captcha: Error bad job id to request reCaptcha.")
+
+        def _checkRequest(response):
+            if response.status_code in [200, 303] and response.json().get('status') == 1:
+                return response
+
+            self.checkErrorStatus(response, 'res.php')
+
+            return None
+
+        response = polling.poll(
+            lambda: self.session.get(
+                '{}/res.php'.format(self.host),
+                params={
+                    'key': self.api_key,
+                    'action': 'get',
+                    'id': jobID,
+                    'json': '1'
+                }
+            ),
+            check_success=_checkRequest,
+            step=5,
+            timeout=180
+        )
+
+        if response:
+            return response.json().get('request')
+        else:
+            raise RuntimeError("2Captcha: Error failed to solve reCaptcha.")
+
+    # ------------------------------------------------------------------------------- #
+
+    def requestSolve(self, site_url, site_key):
+        def _checkRequest(response):
+            if response.status_code in [200, 303] and response.json().get("status") == 1 and response.json().get('request'):
+                return response
+
+            self.checkErrorStatus(response, 'in.php')
+
+            return None
+
+        response = polling.poll(
+            lambda: self.session.post(
+                '{}/in.php'.format(self.host),
+                data={
+                    'key': self.api_key,
+                    'method': 'userrecaptcha',
+                    'googlekey': site_key,
+                    'pageurl': site_url,
+                    'json': '1',
+                    'soft_id': '5507698'
+                },
+                allow_redirects=False
+            ),
+            check_success=_checkRequest,
+            step=5,
+            timeout=180
+        )
+
+        if response:
+            return response.json().get('request')
+        else:
+            raise RuntimeError('2Captcha: Error no job id was returned.')
+
+    # ------------------------------------------------------------------------------- #
+
+    def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
+        jobID = None
+
+        if not reCaptchaParams.get('api_key'):
+            raise ValueError("2Captcha: Missing api_key parameter.")
+
+        self.api_key = reCaptchaParams.get('api_key')
+
+        if reCaptchaParams.get('proxy'):
+            self.session.proxies = reCaptchaParams.get('proxies')
+
+        try:
+            jobID = self.requestSolve(site_url, site_key)
+            return self.requestJob(jobID)
+        except polling.TimeoutException:
+            try:
+                if jobID:
+                    self.reportJob(jobID)
+            except polling.TimeoutException:
+                raise RuntimeError("2Captcha: reCaptcha solve took to long and also failed reporting the job.")
+
+            raise RuntimeError("2Captcha: reCaptcha solve took to long to execute, aborting.")
+
+
+# ------------------------------------------------------------------------------- #
+
+captchaSolver()

lib/cloudscraper/reCaptcha/__init__.py

@@ -1,4 +1,3 @@
-import re
 import sys
 import logging
 import abc
@@ -8,20 +7,20 @@ if sys.version_info >= (3, 4):
 else:
     ABC = abc.ABCMeta('ABC', (), {})
 
-##########################################################################################################################################################
+# ------------------------------------------------------------------------------- #
-
-BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.'
-
-##########################################################################################################################################################
 
 captchaSolvers = {}
 
+# ------------------------------------------------------------------------------- #
+
 
 class reCaptcha(ABC):
     @abc.abstractmethod
     def __init__(self, name):
         captchaSolvers[name] = self
 
+    # ------------------------------------------------------------------------------- #
+
     @classmethod
     def dynamicImport(cls, name):
         if name not in captchaSolvers:
@@ -35,14 +34,13 @@ class reCaptcha(ABC):
 
         return captchaSolvers[name]
 
+    # ------------------------------------------------------------------------------- #
+
     @abc.abstractmethod
     def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
         pass
 
-    def solveCaptcha(self, ret, reCaptchaParams):
+    # ------------------------------------------------------------------------------- #
-        try:
-            site_key = re.search('data-sitekey="(.+?)"', ret.text).group(1)
-        except Exception as e:
-            raise ValueError("Unable to parse Cloudflare\'s reCaptcha variable 'data-sitekey': {} {}".format(e.message, BUG_REPORT))
 
-        return self.getCaptchaAnswer(ret.url, site_key, reCaptchaParams)
+    def solveCaptcha(self, site_url, site_key, reCaptchaParams):
+        return self.getCaptchaAnswer(site_url, site_key, reCaptchaParams)

lib/cloudscraper/reCaptcha/anticaptcha.py

@@ -3,10 +3,10 @@ from __future__ import absolute_import
 import sys
 
 try:
-    from python_anticaptcha import AnticaptchaClient, NoCaptchaTaskProxylessTask, NoCaptchaTask, Proxy
+    from python_anticaptcha import AnticaptchaClient, NoCaptchaTaskProxylessTask
 except ImportError:
     sys.tracebacklimit = 0
-    raise RuntimeError("Please install the python module 'python_anticaptcha' via pip or download it https://github.com/ad-m/python-anticaptcha")
+    raise RuntimeError("Please install the python module 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha")
 
 from . import reCaptcha
 
@@ -22,20 +22,16 @@ class captchaSolver(reCaptcha):
 
         client = AnticaptchaClient(reCaptchaParams.get('api_key'))
 
-        if reCaptchaParams.get('proxy', False) and reCaptchaParams.get('proxies'):
+        if reCaptchaParams.get('proxy'):
             client.session.proxies = reCaptchaParams.get('proxies')
-            task = NoCaptchaTask(
+
-                site_url,
-                site_key,
-                proxy=Proxy.parse_url(
-                    reCaptchaParams.get('proxies').get('https')
-                )
-            )
-        else:
         task = NoCaptchaTaskProxylessTask(site_url, site_key)
 
-        job = client.createTask(task)
+        if not hasattr(client, 'createTaskSmee'):
-        job.join()
+            sys.tracebacklimit = 0
+            raise RuntimeError("Please upgrade 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha")
+
+        job = client.createTaskSmee(task)
         return job.get_solution_response()
 
 

lib/cloudscraper/reCaptcha/deathbycaptcha.py

@@ -20,9 +20,10 @@ class captchaSolver(reCaptcha):
         self.host = 'http://api.dbcapi.me/api'
         self.session = requests.Session()
 
-    ##########################################################################################################################################################
+        # ------------------------------------------------------------------------------- #
 
-    def checkErrorStatus(self, response):
+    @staticmethod
+    def checkErrorStatus(response):
         errors = dict(
             [
                 (400, "DeathByCaptcha: 400 Bad Request"),
@@ -35,7 +36,7 @@ class captchaSolver(reCaptcha):
         if response.status_code in errors:
             raise RuntimeError(errors.get(response.status_code))
 
-    ##########################################################################################################################################################
+        # ------------------------------------------------------------------------------- #
 
     def login(self, username, password):
         self.username = username
@@ -71,7 +72,7 @@ class captchaSolver(reCaptcha):
 
         self.debugRequest(response)
 
-    ##########################################################################################################################################################
+        # ------------------------------------------------------------------------------- #
 
     def reportJob(self, jobID):
         if not jobID:
@@ -104,7 +105,7 @@ class captchaSolver(reCaptcha):
         else:
             raise RuntimeError("DeathByCaptcha: Error report failed reCaptcha.")
 
-    ##########################################################################################################################################################
+        # ------------------------------------------------------------------------------- #
 
     def requestJob(self, jobID):
         if not jobID:
@@ -133,7 +134,7 @@ class captchaSolver(reCaptcha):
         else:
             raise RuntimeError("DeathByCaptcha: Error failed to solve reCaptcha.")
 
-    ##########################################################################################################################################################
+        # ------------------------------------------------------------------------------- #
 
     def requestSolve(self, site_url, site_key):
         def _checkRequest(response):
@@ -169,7 +170,7 @@ class captchaSolver(reCaptcha):
         else:
             raise RuntimeError('DeathByCaptcha: Error no job id was returned.')
 
-    ##########################################################################################################################################################
+        # ------------------------------------------------------------------------------- #
 
     def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
         jobID = None
@@ -195,4 +196,6 @@ class captchaSolver(reCaptcha):
             raise RuntimeError("DeathByCaptcha: reCaptcha solve took to long to execute, aborting.")
 
 
+# ------------------------------------------------------------------------------- #
+
 captchaSolver()

lib/cloudscraper/user_agent/__init__.py

@@ -1,46 +1,82 @@
-import os
 import json
+import os
 import random
-import logging
+import sys
 
 from collections import OrderedDict
 
-##########################################################################################################################################################
+# ------------------------------------------------------------------------------- #
 
 
 class User_Agent():
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
 
     def __init__(self, *args, **kwargs):
         self.headers = None
+        self.cipherSuite = []
         self.loadUserAgent(*args, **kwargs)
 
-    ##########################################################################################################################################################
+    # ------------------------------------------------------------------------------- #
+
+    def loadHeaders(self, user_agents, user_agent_version):
+        if user_agents.get(self.browser).get('releases').get(user_agent_version).get('headers'):
+            self.headers = user_agents.get(self.browser).get('releases').get(user_agent_version).get('headers')
+        else:
+            self.headers = user_agents.get(self.browser).get('default_headers')
+
+    # ------------------------------------------------------------------------------- #
+
+    def filterAgents(self, releases):
+        filtered = {}
+
+        for release in releases:
+            if self.mobile and releases[release]['User-Agent']['mobile']:
+                filtered[release] = filtered.get(release, []) + releases[release]['User-Agent']['mobile']
+
+            if self.desktop and releases[release]['User-Agent']['desktop']:
+                filtered[release] = filtered.get(release, []) + releases[release]['User-Agent']['desktop']
+
+        return filtered
+
+    # ------------------------------------------------------------------------------- #
 
     def loadUserAgent(self, *args, **kwargs):
-        browser = kwargs.pop('browser', None)
+        self.browser = kwargs.pop('browser', None)
+
+        if isinstance(self.browser, dict):
+            self.desktop = self.browser.get('desktop', True)
+            self.mobile = self.browser.get('mobile', True)
+            self.browser = self.browser.get('browser', None)
+        else:
+            self.desktop = kwargs.pop('desktop', True)
+            self.mobile = kwargs.pop('mobile', True)
+
+        if not self.desktop and not self.mobile:
+            sys.tracebacklimit = 0
+            raise RuntimeError("Sorry you can't have mobile and desktop disabled at the same time.")
 
         user_agents = json.load(
             open(os.path.join(os.path.dirname(__file__), 'browsers.json'), 'r'),
             object_pairs_hook=OrderedDict
         )
 
-        if browser and not user_agents.get(browser):
+        if self.browser and not user_agents.get(self.browser):
-            logging.error('Sorry "{}" browser User-Agent was not found.'.format(browser))
+            sys.tracebacklimit = 0
-            raise
+            raise RuntimeError('Sorry "{}" browser User-Agent was not found.'.format(self.browser))
 
-        if not browser:
+        if not self.browser:
-            browser = random.SystemRandom().choice(list(user_agents))
+            self.browser = random.SystemRandom().choice(list(user_agents))
 
-        user_agent_version = random.SystemRandom().choice(list(user_agents.get(browser).get('releases')))
+        self.cipherSuite = user_agents.get(self.browser).get('cipherSuite', [])
 
-        if user_agents.get(browser).get('releases').get(user_agent_version).get('headers'):
+        filteredAgents = self.filterAgents(user_agents.get(self.browser).get('releases'))
-            self.headers = user_agents.get(browser).get('releases').get(user_agent_version).get('headers')
-        else:
-            self.headers = user_agents.get(browser).get('default_headers')
 
-        self.headers['User-Agent'] = random.SystemRandom().choice(user_agents.get(browser).get('releases').get(user_agent_version).get('User-Agent'))
+        user_agent_version = random.SystemRandom().choice(list(filteredAgents))
+
+        self.loadHeaders(user_agents, user_agent_version)
+
+        self.headers['User-Agent'] = random.SystemRandom().choice(filteredAgents[user_agent_version])
 
         if not kwargs.get('allow_brotli', False):
             if 'br' in self.headers['Accept-Encoding']: