stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

update cloudscraper

Browse Source
This commit is contained in:
marco
2019-12-14 11:45:18 +01:00
parent 4f6d2611e3
commit 71aaa90a3d
2 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cloudscraper/__init__.py
+12 -8
View File
@@ -1,3 +1,4 @@
# https://github.com/VeNoMouS/cloudscraper
import logging import logging
import re import re
import sys import sys
@@ -37,7 +38,7 @@ except ImportError:
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
__version__ = '1.2.15' __version__ = '1.2.16'
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
@@ -85,7 +86,7 @@ class CloudScraper(Session):
self.debug = kwargs.pop('debug', False) self.debug = kwargs.pop('debug', False)
self.delay = kwargs.pop('delay', None) self.delay = kwargs.pop('delay', None)
self.cipherSuite = kwargs.pop('cipherSuite', None) self.cipherSuite = kwargs.pop('cipherSuite', None)
self.interpreter = kwargs.pop('interpreter', 'js2py') self.interpreter = kwargs.pop('interpreter', 'native')
self.recaptcha = kwargs.pop('recaptcha', {}) self.recaptcha = kwargs.pop('recaptcha', {})
self.allow_brotli = kwargs.pop( self.allow_brotli = kwargs.pop(
'allow_brotli', 'allow_brotli',
@@ -194,7 +195,7 @@ class CloudScraper(Session):
resp = self.Challenge_Response(resp, **kwargs) resp = self.Challenge_Response(resp, **kwargs)
else: else:
if resp.status_code not in [302, 429, 503]: if not resp.is_redirect and resp.status_code not in [429, 503]:
self._solveDepthCnt = 0 self._solveDepthCnt = 0
return resp return resp
@@ -452,9 +453,7 @@ class CloudScraper(Session):
cloudflare_kwargs['headers'] = updateAttr( cloudflare_kwargs['headers'] = updateAttr(
cloudflare_kwargs, cloudflare_kwargs,
'headers', 'headers',
{ {'Referer': resp.url}
'Referer': resp.url
}
) )
ret = self.request( ret = self.request(
@@ -463,11 +462,16 @@ class CloudScraper(Session):
**cloudflare_kwargs **cloudflare_kwargs
) )
if ret.status_code != 302: # ------------------------------------------------------------------------------- #
# Return response if Cloudflare is doing content pass through instead of 3xx
# ------------------------------------------------------------------------------- #
if not ret.is_redirect:
return ret return ret
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
# We shouldn't be here.... Re-request the original query and process again.... # Cloudflare is doing http 3xx instead of pass through again....
# Re-request the original query and/or process again....
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
return self.request(resp.request.method, resp.url, **kwargs) return self.request(resp.request.method, resp.url, **kwargs)
lib/cloudscraper/user_agent/__init__.py
+4 -4
View File
@@ -5,7 +5,6 @@ import re
import sys import sys
import ssl import ssl
from collections import OrderedDict from collections import OrderedDict
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
@@ -111,6 +110,7 @@ class User_Agent():
self.headers['User-Agent'] = random.SystemRandom().choice(filteredAgents[user_agent_version]) self.headers['User-Agent'] = random.SystemRandom().choice(filteredAgents[user_agent_version])
if not kwargs.get('allow_brotli', False): if not kwargs.get('allow_brotli', False) and 'br' in self.headers['Accept-Encoding']:
if 'br' in self.headers['Accept-Encoding']: self.headers['Accept-Encoding'] = ','.join([
self.headers['Accept-Encoding'] = ','.join([encoding for encoding in self.headers['Accept-Encoding'].split(',') if encoding.strip() != 'br']).strip() encoding for encoding in self.headers['Accept-Encoding'].split(',') if encoding.strip() != 'br'
]).strip()