stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

update cloudscraper

Browse Source
This commit is contained in:
marco
2019-12-14 11:45:18 +01:00
parent 4f6d2611e3
commit 71aaa90a3d
2 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/cloudscraper/__init__.py
View File

@@ -1,3 +1,4 @@
# https://github.com/VeNoMouS/cloudscraper
import logging
import re
import sys
@@ -37,7 +38,7 @@ except ImportError:
# ------------------------------------------------------------------------------- #
__version__ = '1.2.15'
__version__ = '1.2.16'
# ------------------------------------------------------------------------------- #
@@ -85,7 +86,7 @@ class CloudScraper(Session):
self.debug = kwargs.pop('debug', False)
self.delay = kwargs.pop('delay', None)
self.cipherSuite = kwargs.pop('cipherSuite', None)
self.interpreter = kwargs.pop('interpreter', 'js2py')
self.interpreter = kwargs.pop('interpreter', 'native')
self.recaptcha = kwargs.pop('recaptcha', {})
self.allow_brotli = kwargs.pop(
'allow_brotli',
@@ -194,7 +195,7 @@ class CloudScraper(Session):
resp = self.Challenge_Response(resp, **kwargs)
else:
if resp.status_code not in [302, 429, 503]:
if not resp.is_redirect and resp.status_code not in [429, 503]:
self._solveDepthCnt = 0
return resp
@@ -452,9 +453,7 @@ class CloudScraper(Session):
cloudflare_kwargs['headers'] = updateAttr(
cloudflare_kwargs,
'headers',
{
'Referer': resp.url
}
{'Referer': resp.url}
)
ret = self.request(
@@ -463,11 +462,16 @@ class CloudScraper(Session):
**cloudflare_kwargs
)
if ret.status_code != 302:
# ------------------------------------------------------------------------------- #
# Return response if Cloudflare is doing content pass through instead of 3xx
# ------------------------------------------------------------------------------- #
if not ret.is_redirect:
return ret
# ------------------------------------------------------------------------------- #
# We shouldn't be here.... Re-request the original query and process again....
# Cloudflare is doing http 3xx instead of pass through again....
# Re-request the original query and/or process again....
# ------------------------------------------------------------------------------- #
return self.request(resp.request.method, resp.url, **kwargs)

8
lib/cloudscraper/user_agent/__init__.py
View File

@@ -5,7 +5,6 @@ import re
import sys
import ssl
from collections import OrderedDict
# ------------------------------------------------------------------------------- #
@@ -111,6 +110,7 @@ class User_Agent():
self.headers['User-Agent'] = random.SystemRandom().choice(filteredAgents[user_agent_version])
if not kwargs.get('allow_brotli', False):
if 'br' in self.headers['Accept-Encoding']:
self.headers['Accept-Encoding'] = ','.join([encoding for encoding in self.headers['Accept-Encoding'].split(',') if encoding.strip() != 'br']).strip()
if not kwargs.get('allow_brotli', False) and 'br' in self.headers['Accept-Encoding']:
self.headers['Accept-Encoding'] = ','.join([
encoding for encoding in self.headers['Accept-Encoding'].split(',') if encoding.strip() != 'br'
]).strip()