diff --git a/lib/cloudscraper/__init__.py b/lib/cloudscraper/__init__.py index 31a4cd36..3a056269 100644 --- a/lib/cloudscraper/__init__.py +++ b/lib/cloudscraper/__init__.py @@ -54,7 +54,7 @@ except ImportError: # ------------------------------------------------------------------------------- # -__version__ = '1.2.32' +__version__ = '1.2.34' # ------------------------------------------------------------------------------- # @@ -102,6 +102,7 @@ class CloudScraper(Session): self.debug = kwargs.pop('debug', False) self.delay = kwargs.pop('delay', None) self.cipherSuite = kwargs.pop('cipherSuite', None) + self.ssl_context = kwargs.pop('ssl_context', None) self.interpreter = kwargs.pop('interpreter', 'native') self.recaptcha = kwargs.pop('recaptcha', {}) self.allow_brotli = kwargs.pop( @@ -134,7 +135,8 @@ class CloudScraper(Session): self.mount( 'https://', CipherSuiteAdapter( - cipherSuite=self.cipherSuite + cipherSuite=self.cipherSuite, + ssl_context=self.ssl_context ) ) @@ -253,7 +255,7 @@ class CloudScraper(Session): resp.headers.get('Server', '').startswith('cloudflare') and resp.status_code in [429, 503] and re.search( - r'action="/.*?__cf_chl_jschl_tk__=\S+".*?name="jschl_vc"\svalue=.*?', + r'
', formPayload['form']): + inputPayload = dict(re.findall(r'(\S+)="(\S+)"', challengeParam)) + if inputPayload.get('name') in ['r', 'jschl_vc', 'pass']: + payload.update({inputPayload['name']: inputPayload['value']}) except AttributeError: self.simpleException( diff --git a/lib/cloudscraper/exceptions.py b/lib/cloudscraper/exceptions.py index 77ccbc75..525b1743 100644 --- a/lib/cloudscraper/exceptions.py +++ b/lib/cloudscraper/exceptions.py @@ -36,6 +36,12 @@ class CloudflareIUAMError(CloudflareException): """ +class CloudflareSolveError(CloudflareException): + """ + Raise an error when issue with solving Cloudflare challenge + """ + + class CloudflareReCaptchaError(CloudflareException): """ Raise an error for problem extracting reCaptcha paramters diff --git a/lib/cloudscraper/help.py b/lib/cloudscraper/help.py new file mode 100644 index 00000000..24d72e29 --- /dev/null +++ b/lib/cloudscraper/help.py @@ -0,0 +1,79 @@ +import json +import platform +import requests +import ssl +import sys +import urllib3 + +from collections import OrderedDict +from . import __version__ as cloudscraper_version + +# ------------------------------------------------------------------------------- # + + +def getPossibleCiphers(): + try: + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) + context.set_ciphers('ALL') + return sorted([cipher['name'] for cipher in context.get_ciphers()]) + except AttributeError: + return 'get_ciphers() is unsupported' + +# ------------------------------------------------------------------------------- # + + +def _pythonVersion(): + interpreter = platform.python_implementation() + interpreter_version = platform.python_version() + + if interpreter == 'PyPy': + interpreter_version = '{}.{}.{}'.format( + sys.pypy_version_info.major, + sys.pypy_version_info.minor, + sys.pypy_version_info.micro + ) + if sys.pypy_version_info.releaselevel != 'final': + interpreter_version = '{}{}'.format( + interpreter_version, + sys.pypy_version_info.releaselevel + ) + + return { + 'name': interpreter, + 'version': interpreter_version + } + +# ------------------------------------------------------------------------------- # + + +def systemInfo(): + try: + platform_info = { + 'system': platform.system(), + 'release': platform.release(), + } + except IOError: + platform_info = { + 'system': 'Unknown', + 'release': 'Unknown', + } + + return OrderedDict([ + ('platform', platform_info), + ('interpreter', _pythonVersion()), + ('cloudscraper', cloudscraper_version), + ('requests', requests.__version__), + ('urllib3', urllib3.__version__), + ('OpenSSL', OrderedDict( + [ + ('version', ssl.OPENSSL_VERSION), + ('ciphers', getPossibleCiphers()) + ] + )) + ]) + +# ------------------------------------------------------------------------------- # + + +if __name__ == '__main__': + print(json.dumps(systemInfo(), indent=4)) diff --git a/lib/cloudscraper/interpreters/__init__.py b/lib/cloudscraper/interpreters/__init__.py index d31f371f..10955552 100644 --- a/lib/cloudscraper/interpreters/__init__.py +++ b/lib/cloudscraper/interpreters/__init__.py @@ -2,6 +2,8 @@ import sys import logging import abc +from ..exceptions import CloudflareSolveError + if sys.version_info >= (3, 4): ABC = abc.ABC # noqa else: @@ -10,7 +12,6 @@ else: # ------------------------------------------------------------------------------- # interpreters = {} -BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.' # ------------------------------------------------------------------------------- # @@ -50,5 +51,6 @@ class JavaScriptInterpreter(ABC): try: return float(self.eval(body, domain)) except Exception: - logging.error('Error executing Cloudflare IUAM Javascript. {}'.format(BUG_REPORT)) - raise + raise CloudflareSolveError( + 'Error trying to solve Cloudflare IUAM Javascript, they may have changed their technique.' + ) diff --git a/lib/cloudscraper/reCaptcha/anticaptcha.py b/lib/cloudscraper/reCaptcha/anticaptcha.py index 3c45abe0..4a95111f 100644 --- a/lib/cloudscraper/reCaptcha/anticaptcha.py +++ b/lib/cloudscraper/reCaptcha/anticaptcha.py @@ -1,5 +1,4 @@ from __future__ import absolute_import - from ..exceptions import ( reCaptchaParameter, reCaptchaTimeout, @@ -19,12 +18,19 @@ except ImportError: "pip install python-anticaptcha or https://github.com/ad-m/python-anticaptcha/" ) +import sys + from . import reCaptcha class captchaSolver(reCaptcha): def __init__(self): + if sys.modules['python_anticaptcha'].__version__ < 0.6: + raise ImportError( + "Please upgrade the python module 'python_anticaptcha' via " + "pip install -U python-anticaptcha or https://github.com/ad-m/python-anticaptcha/" + ) super(captchaSolver, self).__init__('anticaptcha') # ------------------------------------------------------------------------------- # @@ -51,7 +57,7 @@ class captchaSolver(reCaptcha): "https://github.com/ad-m/python-anticaptcha/tree/hcaptcha" ) - job = client.createTaskSmee(task) + job = client.createTaskSmee(task, timeout=180) try: job.join(maximum_time=180) diff --git a/lib/cloudscraper/user_agent/browsers.json b/lib/cloudscraper/user_agent/browsers.json index 54a69541..5a572168 100644 --- a/lib/cloudscraper/user_agent/browsers.json +++ b/lib/cloudscraper/user_agent/browsers.json @@ -2,7 +2,6 @@ "chrome": { "default_headers": { "User-Agent": null, - "DNT": "1", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9", "Accept-Encoding": "gzip, deflate, br" @@ -17,9 +16,13 @@ "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-CHACHA20-POLY1305", "ECDHE-RSA-CHACHA20-POLY1305", + "ECDHE-RSA-AES128-SHA", + "ECDHE-RSA-AES256-SHA", "AES128-GCM-SHA256", "AES256-GCM-SHA384", - "AES128-SHA" + "AES128-SHA", + "AES256-SHA", + "DES-CBC3-SHA" ], "releases": { "Chrome/50.0.0.0": { @@ -12807,8 +12810,7 @@ "User-Agent": null, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate, br", - "DNT": "1" + "Accept-Encoding": "gzip, deflate, br" }, "cipherSuite": [ "TLS_AES_128_GCM_SHA256", @@ -12822,9 +12824,13 @@ "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA", "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", + "ECDHE-RSA-AES256-SHA", "DHE-RSA-AES128-SHA", "DHE-RSA-AES256-SHA", - "AES128-SHA" + "AES128-SHA", + "AES256-SHA", + "DES-CBC3-SHA" ], "releases": { "Firefox/50.0": {