stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Fix Cloudscraper

Browse Source 
ritornati alla vecchia versione di js2py
modificata versione 1.1.24 di cloudscrape
This commit is contained in:
Alhaziel
2019-12-02 11:35:50 +01:00
parent 89764d0a7d
commit 6abff904d4
19 changed files with 11392 additions and 13822 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cloudscraper/__init__.py
+150 -229
View File
@@ -1,20 +1,20 @@
## Modded version of cloudscrape 1.1.24
## https://github.com/venomous/cloudscraper
import logging import logging
import re import re
import sys import sys
import ssl import ssl
import requests import requests
try:
import copyreg
except ImportError:
import copy_reg as copyreg
from copy import deepcopy from copy import deepcopy
from time import sleep from time import sleep
from collections import OrderedDict from collections import OrderedDict
from requests.sessions import Session from requests.sessions import Session
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.ssl_ import create_urllib3_context
from .interpreters import JavaScriptInterpreter from .interpreters import JavaScriptInterpreter
from .reCaptcha import reCaptcha from .reCaptcha import reCaptcha
@@ -35,210 +35,153 @@ try:
except ImportError: except ImportError:
from urllib.parse import urlparse from urllib.parse import urlparse
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
__version__ = '1.2.14' __version__ = '1.1.24'
# ------------------------------------------------------------------------------- # BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.'
##########################################################################################################################################################
# class CipherSuiteAdapter(HTTPAdapter):
#
# def __init__(self, cipherSuite=None, **kwargs):
# self.cipherSuite = cipherSuite
#
# self.ssl_context = create_urllib3_context(
# ssl_version=ssl.PROTOCOL_TLS,
# ciphers=self.cipherSuite
# )
#
# super(CipherSuiteAdapter, self).__init__(**kwargs)
class CipherSuiteAdapter(HTTPAdapter): class CipherSuiteAdapter(HTTPAdapter):
__attrs__ = [ def __init__(self, cipherSuite=None, **kwargs):
'ssl_context', self.cipherSuite = cipherSuite
'max_retries',
'config',
'_pool_connections',
'_pool_maxsize',
'_pool_block'
]
def __init__(self, *args, **kwargs): if hasattr(ssl, 'PROTOCOL_TLS'):
self.ssl_context = kwargs.pop('ssl_context', None) self.ssl_context = create_urllib3_context(
self.cipherSuite = kwargs.pop('cipherSuite', None) ssl_version=getattr(ssl, 'PROTOCOL_TLSv1_3', ssl.PROTOCOL_TLSv1_2),
ciphers=self.cipherSuite
if not self.ssl_context: )
self.ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) else:
self.ssl_context.set_ciphers(self.cipherSuite) self.ssl_context = create_urllib3_context(ssl_version=ssl.PROTOCOL_TLSv1)
self.ssl_context.options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1)
super(CipherSuiteAdapter, self).__init__(**kwargs) super(CipherSuiteAdapter, self).__init__(**kwargs)
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def init_poolmanager(self, *args, **kwargs): def init_poolmanager(self, *args, **kwargs):
kwargs['ssl_context'] = self.ssl_context kwargs['ssl_context'] = self.ssl_context
return super(CipherSuiteAdapter, self).init_poolmanager(*args, **kwargs) return super(CipherSuiteAdapter, self).init_poolmanager(*args, **kwargs)
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def proxy_manager_for(self, *args, **kwargs): def proxy_manager_for(self, *args, **kwargs):
kwargs['ssl_context'] = self.ssl_context kwargs['ssl_context'] = self.ssl_context
return super(CipherSuiteAdapter, self).proxy_manager_for(*args, **kwargs) return super(CipherSuiteAdapter, self).proxy_manager_for(*args, **kwargs)
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
class CloudScraper(Session): class CloudScraper(Session):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.allow_brotli = kwargs.pop('allow_brotli', True if 'brotli' in sys.modules.keys() else False)
self.debug = kwargs.pop('debug', False) self.debug = kwargs.pop('debug', False)
self.delay = kwargs.pop('delay', None) self.delay = kwargs.pop('delay', None)
self.cipherSuite = kwargs.pop('cipherSuite', None)
self.interpreter = kwargs.pop('interpreter', 'js2py') self.interpreter = kwargs.pop('interpreter', 'js2py')
self.recaptcha = kwargs.pop('recaptcha', {}) self.recaptcha = kwargs.pop('recaptcha', {})
self.allow_brotli = kwargs.pop(
'allow_brotli',
True if 'brotli' in sys.modules.keys() else False
)
self.user_agent = User_Agent(
allow_brotli=self.allow_brotli,
browser=kwargs.pop('browser', None)
)
self._solveDepthCnt = 0 self.cipherSuite = None
self.solveDepth = kwargs.pop('solveDepth', 3)
super(CloudScraper, self).__init__(*args, **kwargs) super(CloudScraper, self).__init__(*args, **kwargs)
# pylint: disable=E0203
if 'requests' in self.headers['User-Agent']: if 'requests' in self.headers['User-Agent']:
# ------------------------------------------------------------------------------- #
# Set a random User-Agent if no custom User-Agent has been set # Set a random User-Agent if no custom User-Agent has been set
# ------------------------------------------------------------------------------- # self.headers = User_Agent(allow_brotli=self.allow_brotli).headers
self.headers = self.user_agent.headers
self.mount( self.mount('https://', CipherSuiteAdapter(self.loadCipherSuite()))
'https://',
CipherSuiteAdapter(
cipherSuite=self.loadCipherSuite() if not self.cipherSuite else self.cipherSuite
)
)
# purely to allow us to pickle dump ##########################################################################################################################################################
copyreg.pickle(ssl.SSLContext, lambda obj: (obj.__class__, (obj.protocol,)))
# ------------------------------------------------------------------------------- #
# Allow us to pickle our session back with all variables
# ------------------------------------------------------------------------------- #
def __getstate__(self):
return self.__dict__
# ------------------------------------------------------------------------------- #
# debug the request via the response
# ------------------------------------------------------------------------------- #
@staticmethod @staticmethod
def debugRequest(req): def debugRequest(req):
try: try:
print(dump.dump_all(req).decode('utf-8')) print(dump.dump_all(req).decode('utf-8'))
except ValueError as e: except: # noqa
print("Debug Error: {}".format(getattr(e, 'message', e))) pass
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
# Decode Brotli on older versions of urllib3 manually
# ------------------------------------------------------------------------------- #
def decodeBrotli(self, resp):
if requests.packages.urllib3.__version__ < '1.25.1' and resp.headers.get('Content-Encoding') == 'br':
if self.allow_brotli and resp._content:
resp._content = brotli.decompress(resp.content)
else:
logging.warning(
'You\'re running urllib3 {}, Brotli content detected, '
'Which requires manual decompression, '
'But option allow_brotli is set to False, '
'We will not continue to decompress.'.format(requests.packages.urllib3.__version__)
)
return resp
# ------------------------------------------------------------------------------- #
# construct a cipher suite of ciphers the system actually supports
# ------------------------------------------------------------------------------- #
def loadCipherSuite(self): def loadCipherSuite(self):
if self.cipherSuite: if self.cipherSuite:
return self.cipherSuite return self.cipherSuite
if hasattr(ssl, 'Purpose') and hasattr(ssl.Purpose, 'SERVER_AUTH'): self.cipherSuite = ''
for cipher in self.user_agent.cipherSuite[:]:
if hasattr(ssl, 'PROTOCOL_TLS'):
ciphers = [
'TLS13-AES-128-GCM-SHA256',
'TLS13-AES-256-GCM-SHA384',
'TLS13-CHACHA20-POLY1305-SHA256',
'ECDHE-ECDSA-CHACHA20-POLY1305',
'ECDHE-ECDSA-AES128-GCM-SHA256',
'ECDHE-ECDSA-AES128-SHA',
'ECDHE-ECDSA-AES128-SHA256',
'ECDHE-ECDSA-AES256-GCM-SHA384',
'ECDHE-ECDSA-AES256-SHA',
'ECDHE-ECDSA-AES256-SHA384',
# Slip in some additional intermediate compatibility ciphers, This should help out users for non Cloudflare based sites.
'ECDHE-RSA-AES128-SHA256',
'ECDHE-RSA-AES256-SHA384',
'ECDHE-RSA-AES256-GCM-SHA384',
'DHE-RSA-AES128-GCM-SHA256',
'DHE-RSA-AES256-GCM-SHA384'
]
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
for cipher in ciphers:
try: try:
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) ctx.set_ciphers(cipher)
context.set_ciphers(cipher) self.cipherSuite = '{}:{}'.format(self.cipherSuite, cipher).rstrip(':').lstrip(':')
except: except ssl.SSLError:
self.user_agent.cipherSuite.remove(cipher) pass
if self.user_agent.cipherSuite: return self.cipherSuite
self.cipherSuite = ':'.join(self.user_agent.cipherSuite)
return self.cipherSuite
sys.tracebacklimit = 0 ##########################################################################################################################################################
raise RuntimeError("The OpenSSL on this system does not meet the minimum cipher requirements.")
# ------------------------------------------------------------------------------- #
# Our hijacker request function
# ------------------------------------------------------------------------------- #
def request(self, method, url, *args, **kwargs): def request(self, method, url, *args, **kwargs):
# pylint: disable=E0203 ourSuper = super(CloudScraper, self)
if kwargs.get('proxies') and kwargs.get('proxies') != self.proxies: resp = ourSuper.request(method, url, *args, **kwargs)
self.proxies = kwargs.get('proxies')
resp = self.decodeBrotli( if requests.packages.urllib3.__version__ < '1.25.1' and resp.headers.get('Content-Encoding') == 'br':
super(CloudScraper, self).request(method, url, *args, **kwargs) if self.allow_brotli and resp._content:
) resp._content = brotli.decompress(resp.content)
else:
logging.warning('Brotli content detected, But option is disabled, we will not continue.')
return resp
# ------------------------------------------------------------------------------- #
# Debug request # Debug request
# ------------------------------------------------------------------------------- #
if self.debug: if self.debug:
self.debugRequest(resp) self.debugRequest(resp)
# Check if Cloudflare anti-bot is on # Check if Cloudflare anti-bot is on
if self.is_Challenge_Request(resp): if self.isChallengeRequest(resp):
# ------------------------------------------------------------------------------- # if resp.request.method != 'GET':
# Try to solve the challenge and send it back # Work around if the initial request is not a GET,
# ------------------------------------------------------------------------------- # # Supersede with a GET then re-request the original METHOD.
self.request('GET', resp.url)
if self._solveDepthCnt >= self.solveDepth: resp = ourSuper.request(method, url, *args, **kwargs)
sys.tracebacklimit = 0 else:
_ = self._solveDepthCnt # Solve Challenge
self._solveDepthCnt = 0 resp = self.sendChallengeResponse(resp, **kwargs)
raise RuntimeError("!!Loop Protection!! We have tried to solve {} time(s) in a row.".format(_))
self._solveDepthCnt += 1
resp = self.Challenge_Response(resp, **kwargs)
else:
if resp.status_code not in [302, 429, 503]:
self._solveDepthCnt = 0
return resp return resp
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
# check if the response contains a valid Cloudflare challenge
# ------------------------------------------------------------------------------- #
@staticmethod
def is_IUAM_Challenge(resp):
try:
return (
resp.headers.get('Server', '').startswith('cloudflare')
and resp.status_code in [429, 503]
and re.search(
r'action="/.*?__cf_chl_jschl_tk__=\S+".*?name="jschl_vc"\svalue=.*?',
resp.text,
re.M | re.DOTALL
)
)
except AttributeError:
pass
return False
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
# check if the response contains a valid Cloudflare reCaptcha challenge # check if the response contains a valid Cloudflare reCaptcha challenge
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
@@ -260,40 +203,41 @@ class CloudScraper(Session):
return False return False
# ------------------------------------------------------------------------------- # @staticmethod
# Wrapper for is_reCaptcha_Challenge and is_IUAM_Challenge def isChallengeRequest(resp):
# ------------------------------------------------------------------------------- # if resp.headers.get('Server', '').startswith('cloudflare'):
return (
def is_Challenge_Request(self, resp): resp.status_code in [403, 429, 503]
if self.is_reCaptcha_Challenge(resp) or self.is_IUAM_Challenge(resp): and (
return True all(s in resp.content for s in [b'jschl_vc', b'jschl_answer'])
or
all(s in resp.content for s in [b'why_captcha', b'/cdn-cgi/l/chk_captcha'])
)
)
return False return False
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
# Try to solve cloudflare javascript challenge. # Try to solve cloudflare javascript challenge.
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
@staticmethod @staticmethod
def IUAM_Challenge_Response(body, url, interpreter): def IUAM_Challenge_Response(body, domain, interpreter):
try: try:
challengeUUID = re.search( challengeUUID = re.search(
r'id="challenge-form" action="(?P<challengeUUID>\S+)"', r'__cf_chl_jschl_tk__=(?P<challengeUUID>\S+)"',
body, re.M | re.DOTALL body, re.M | re.DOTALL
).groupdict().get('challengeUUID', '') ).groupdict().get('challengeUUID')
payload = OrderedDict(re.findall(r'name="(r|jschl_vc|pass)"\svalue="(.*?)"', body)) params = OrderedDict(re.findall(r'name="(r|jschl_vc|pass)"\svalue="(.*?)"', body))
except AttributeError: except AttributeError:
sys.tracebacklimit = 0 sys.tracebacklimit = 0
raise RuntimeError( raise RuntimeError("Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly.")
"Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly."
)
hostParsed = urlparse(url)
try: try:
payload['jschl_answer'] = JavaScriptInterpreter.dynamicImport( params['jschl_answer'] = JavaScriptInterpreter.dynamicImport(
interpreter interpreter
).solveChallenge(body, hostParsed.netloc) ).solveChallenge(body, domain)
except Exception as e: except Exception as e:
raise RuntimeError( raise RuntimeError(
'Unable to parse Cloudflare anti-bots page: {}'.format( 'Unable to parse Cloudflare anti-bots page: {}'.format(
@@ -302,23 +246,16 @@ class CloudScraper(Session):
) )
return { return {
'url': '{}://{}{}'.format( 'url': 'https://{}/'.format(domain),
hostParsed.scheme, 'params': {'__cf_chl_jschl_tk__': challengeUUID},
hostParsed.netloc, 'data': params
challengeUUID
),
'data': payload
} }
# ------------------------------------------------------------------------------- #
# Try to solve the reCaptcha challenge via 3rd party.
# ------------------------------------------------------------------------------- #
@staticmethod @staticmethod
def reCaptcha_Challenge_Response(provider, provider_params, body, url): def reCaptcha_Challenge_Response(provider, provider_params, body, url):
try: try:
payload = re.search( params = re.search(
r'(name="r"\svalue="(?P<r>\S+)"|).*?challenge-form" action="(?P<challengeUUID>\S+)".*?' r'(name="r"\svalue="(?P<r>\S+)"|).*?__cf_chl_captcha_tk__=(?P<challengeUUID>\S+)".*?'
r'data-ray="(?P<data_ray>\S+)".*?data-sitekey="(?P<site_key>\S+)"', r'data-ray="(?P<data_ray>\S+)".*?data-sitekey="(?P<site_key>\S+)"',
body, re.M | re.DOTALL body, re.M | re.DOTALL
).groupdict() ).groupdict()
@@ -328,30 +265,24 @@ class CloudScraper(Session):
"Cloudflare reCaptcha detected, unfortunately we can't extract the parameters correctly." "Cloudflare reCaptcha detected, unfortunately we can't extract the parameters correctly."
) )
hostParsed = urlparse(url)
return { return {
'url': '{}://{}{}'.format( 'url': url,
hostParsed.scheme, 'params': {'__cf_chl_captcha_tk__': params.get('challengeUUID')},
hostParsed.netloc,
payload.get('challengeUUID', '')
),
'data': OrderedDict([ 'data': OrderedDict([
('r', payload.get('r', '')), ('r', ''),
('id', payload.get('data_ray')), ('id', params.get('data_ray')),
( (
'g-recaptcha-response', 'g-recaptcha-response',
reCaptcha.dynamicImport( reCaptcha.dynamicImport(
provider.lower() provider.lower()
).solveCaptcha(url, payload.get('site_key'), provider_params) ).solveCaptcha(url, params.get('site_key'), provider_params)
) )
]) ])
} }
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
# Attempt to handle and send the challenge response back to cloudflare
# ------------------------------------------------------------------------------- #
def Challenge_Response(self, resp, **kwargs): def sendChallengeResponse(self, resp, **original_kwargs):
if self.is_reCaptcha_Challenge(resp): if self.is_reCaptcha_Challenge(resp):
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
# double down on the request as some websites are only checking # double down on the request as some websites are only checking
@@ -359,7 +290,7 @@ class CloudScraper(Session):
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
resp = self.decodeBrotli( resp = self.decodeBrotli(
super(CloudScraper, self).request(resp.request.method, resp.url, **kwargs) super(CloudScraper, self).request(resp.request.method, resp.url, **original_kwargs)
) )
if not self.is_reCaptcha_Challenge(resp): if not self.is_reCaptcha_Challenge(resp):
@@ -415,7 +346,7 @@ class CloudScraper(Session):
submit_url = self.IUAM_Challenge_Response( submit_url = self.IUAM_Challenge_Response(
resp.text, resp.text,
resp.url, urlparse(resp.url).netloc,
self.interpreter self.interpreter
) )
@@ -433,34 +364,33 @@ class CloudScraper(Session):
obj[name].update(newValue) obj[name].update(newValue)
return obj[name] return obj[name]
cloudflare_kwargs = deepcopy(kwargs) cloudflare_kwargs = deepcopy(original_kwargs)
cloudflare_kwargs['allow_redirects'] = False cloudflare_kwargs['allow_redirects'] = False
cloudflare_kwargs['data'] = updateAttr( cloudflare_kwargs['params'] = updateAttr(cloudflare_kwargs, 'params', submit_url['params'])
cloudflare_kwargs, cloudflare_kwargs['data'] = updateAttr(cloudflare_kwargs, 'data', submit_url['data'])
'data', cloudflare_kwargs['headers'] = updateAttr(cloudflare_kwargs, 'headers', {'Referer': resp.url})
submit_url['data']
)
cloudflare_kwargs['headers'] = updateAttr(
cloudflare_kwargs,
'headers',
{
'Referer': resp.url
}
)
return self.request( self.request(
'POST', 'POST',
submit_url['url'], submit_url['url'],
**cloudflare_kwargs **cloudflare_kwargs
) )
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
# We shouldn't be here.... Re-request the original query and process again.... # Request the original query request and return it
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
return self.request(resp.request.method, resp.url, **kwargs) return self.request(resp.request.method, resp.url, **original_kwargs)
# ------------------------------------------------------------------------------- #
# Request the original query request and return it
# ------------------------------------------------------------------------------- #
# return self.request(resp.request.method, resp.url, **kwargs)
# ------------------------------------------------------------------------------- # # ------------------------------------------------------------------------------- #
##########################################################################################################################################################
@classmethod @classmethod
def create_scraper(cls, sess=None, **kwargs): def create_scraper(cls, sess=None, **kwargs):
""" """
@@ -469,30 +399,25 @@ class CloudScraper(Session):
scraper = cls(**kwargs) scraper = cls(**kwargs)
if sess: if sess:
for attr in ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']: attrs = ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']
for attr in attrs:
val = getattr(sess, attr, None) val = getattr(sess, attr, None)
if val: if val:
setattr(scraper, attr, val) setattr(scraper, attr, val)
return scraper return scraper
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
# Functions for integrating cloudscraper with other applications and scripts
# ------------------------------------------------------------------------------- #
# Functions for integrating cloudscraper with other applications and scripts
@classmethod @classmethod
def get_tokens(cls, url, **kwargs): def get_tokens(cls, url, **kwargs):
scraper = cls.create_scraper( scraper = cls.create_scraper(
**{ debug=kwargs.pop('debug', False),
field: kwargs.pop(field, None) for field in [ delay=kwargs.pop('delay', None),
'allow_brotli', interpreter=kwargs.pop('interpreter', 'js2py'),
'browser', allow_brotli=kwargs.pop('allow_brotli', True),
'debug', recaptcha=kwargs.pop('recaptcha', {})
'delay',
'interpreter',
'recaptcha'
] if field in kwargs
}
) )
try: try:
@@ -511,11 +436,7 @@ class CloudScraper(Session):
cookie_domain = d cookie_domain = d
break break
else: else:
sys.tracebacklimit = 0 raise ValueError('Unable to find Cloudflare cookies. Does the site actually have Cloudflare IUAM ("I\'m Under Attack Mode") enabled?')
raise RuntimeError(
"Unable to find Cloudflare cookies. Does the site actually "
"have Cloudflare IUAM (I'm Under Attack Mode) enabled?"
)
return ( return (
{ {
@@ -525,7 +446,7 @@ class CloudScraper(Session):
scraper.headers['User-Agent'] scraper.headers['User-Agent']
) )
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
@classmethod @classmethod
def get_cookie_string(cls, url, **kwargs): def get_cookie_string(cls, url, **kwargs):
@@ -536,7 +457,7 @@ class CloudScraper(Session):
return '; '.join('='.join(pair) for pair in tokens.items()), user_agent return '; '.join('='.join(pair) for pair in tokens.items()), user_agent
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
create_scraper = CloudScraper.create_scraper create_scraper = CloudScraper.create_scraper
get_tokens = CloudScraper.get_tokens get_tokens = CloudScraper.get_tokens
lib/cloudscraper/interpreters/js2py.py
+3
View File
@@ -15,6 +15,9 @@ class ChallengeInterpreter(JavaScriptInterpreter):
super(ChallengeInterpreter, self).__init__('js2py') super(ChallengeInterpreter, self).__init__('js2py')
def eval(self, jsEnv, js): def eval(self, jsEnv, js):
### blocca lo script
# from core.support import dbg; dbg()
if js2py.eval_js('(+(+!+[]+[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+[!+[]+!+[]]+[+[]])+[])[+!+[]]') == '1': if js2py.eval_js('(+(+!+[]+[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+[!+[]+!+[]]+[+[]])+[])[+!+[]]') == '1':
logging.warning('WARNING - Please upgrade your js2py https://github.com/PiotrDabkowski/Js2Py, applying work around for the meantime.') logging.warning('WARNING - Please upgrade your js2py https://github.com/PiotrDabkowski/Js2Py, applying work around for the meantime.')
js = jsunfuck(js) js = jsunfuck(js)
lib/cloudscraper/interpreters/nodejs.py
+9 -5
View File
@@ -1,10 +1,14 @@
import base64 import base64
import logging
import subprocess import subprocess
import sys
from . import JavaScriptInterpreter from . import JavaScriptInterpreter
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.'
##########################################################################################################################################################
class ChallengeInterpreter(JavaScriptInterpreter): class ChallengeInterpreter(JavaScriptInterpreter):
@@ -33,10 +37,10 @@ class ChallengeInterpreter(JavaScriptInterpreter):
) )
raise raise
except Exception: except Exception:
sys.tracebacklimit = 0 logging.error('Error executing Cloudflare IUAM Javascript. %s' % BUG_REPORT)
raise RuntimeError('Error executing Cloudflare IUAM Javascript in nodejs') raise
pass
# ------------------------------------------------------------------------------- #
ChallengeInterpreter() ChallengeInterpreter()
lib/cloudscraper/interpreters/v8.py
+1 -7
View File
@@ -10,23 +10,17 @@ except ImportError:
from . import JavaScriptInterpreter from . import JavaScriptInterpreter
# ------------------------------------------------------------------------------- #
class ChallengeInterpreter(JavaScriptInterpreter): class ChallengeInterpreter(JavaScriptInterpreter):
def __init__(self): def __init__(self):
super(ChallengeInterpreter, self).__init__('v8') super(ChallengeInterpreter, self).__init__('v8')
# ------------------------------------------------------------------------------- #
def eval(self, jsEnv, js): def eval(self, jsEnv, js):
try: try:
return v8eval.V8().eval('{}{}'.format(jsEnv, js)) return v8eval.V8().eval('{}{}'.format(jsEnv, js))
except (TypeError, v8eval.V8Error): except: # noqa
RuntimeError('We encountered an error running the V8 Engine.') RuntimeError('We encountered an error running the V8 Engine.')
# ------------------------------------------------------------------------------- #
ChallengeInterpreter() ChallengeInterpreter()
lib/cloudscraper/reCaptcha/2captcha.py
-206
View File
@@ -1,206 +0,0 @@
from __future__ import absolute_import
import requests
try:
import polling
except ImportError:
import sys
sys.tracebacklimit = 0
raise RuntimeError("Please install the python module 'polling' via pip or download it from https://github.com/justiniso/polling/")
from . import reCaptcha
class captchaSolver(reCaptcha):
def __init__(self):
super(captchaSolver, self).__init__('2captcha')
self.host = 'https://2captcha.com'
self.session = requests.Session()
# ------------------------------------------------------------------------------- #
@staticmethod
def checkErrorStatus(response, request_type):
if response.status_code in [500, 502]:
raise RuntimeError('2Captcha: Server Side Error {}'.format(response.status_code))
errors = {
'in.php': {
"ERROR_WRONG_USER_KEY": "You've provided api_key parameter value is in incorrect format, it should contain 32 symbols.",
"ERROR_KEY_DOES_NOT_EXIST": "The api_key you've provided does not exists.",
"ERROR_ZERO_BALANCE": "You don't have sufficient funds on your account.",
"ERROR_PAGEURL": "pageurl parameter is missing in your request.",
"ERROR_NO_SLOT_AVAILABLE":
"No Slots Available.\nYou can receive this error in two cases:\n"
"1. If you solve ReCaptcha: the queue of your captchas that are not distributed to workers is too long. "
"Queue limit changes dynamically and depends on total amount of captchas awaiting solution and usually it's between 50 and 100 captchas.\n"
"2. If you solve Normal Captcha: your maximum rate for normal captchas is lower than current rate on the server."
"You can change your maximum rate in your account's settings.",
"ERROR_IP_NOT_ALLOWED": "The request is sent from the IP that is not on the list of your allowed IPs.",
"IP_BANNED": "Your IP address is banned due to many frequent attempts to access the server using wrong authorization keys.",
"ERROR_BAD_TOKEN_OR_PAGEURL":
"You can get this error code when sending ReCaptcha V2. "
"That happens if your request contains invalid pair of googlekey and pageurl. "
"The common reason for that is that ReCaptcha is loaded inside an iframe hosted on another domain/subdomain.",
"ERROR_GOOGLEKEY":
"You can get this error code when sending ReCaptcha V2. "
"That means that sitekey value provided in your request is incorrect: it's blank or malformed.",
"MAX_USER_TURN": "You made more than 60 requests within 3 seconds.Your account is banned for 10 seconds. Ban will be lifted automatically."
},
'res.php': {
"ERROR_CAPTCHA_UNSOLVABLE":
"We are unable to solve your captcha - three of our workers were unable solve it "
"or we didn't get an answer within 90 seconds (300 seconds for ReCaptcha V2). "
"We will not charge you for that request.",
"ERROR_WRONG_USER_KEY": "You've provided api_key parameter value in incorrect format, it should contain 32 symbols.",
"ERROR_KEY_DOES_NOT_EXIST": "The api_key you've provided does not exists.",
"ERROR_WRONG_ID_FORMAT": "You've provided captcha ID in wrong format. The ID can contain numbers only.",
"ERROR_WRONG_CAPTCHA_ID": "You've provided incorrect captcha ID.",
"ERROR_BAD_DUPLICATES":
"Error is returned when 100% accuracy feature is enabled. "
"The error means that max numbers of tries is reached but min number of matches not found.",
"REPORT_NOT_RECORDED": "Error is returned to your complain request if you already complained lots of correctly solved captchas.",
"ERROR_IP_ADDRES":
"You can receive this error code when registering a pingback (callback) IP or domain."
"That happes if your request is coming from an IP address that doesn't match the IP address of your pingback IP or domain.",
"ERROR_TOKEN_EXPIRED": "You can receive this error code when sending GeeTest. That error means that challenge value you provided is expired.",
"ERROR_EMPTY_ACTION": "Action parameter is missing or no value is provided for action parameter."
}
}
if response.json().get('status') is False and response.json().get('request') in errors.get(request_type):
raise RuntimeError('{} {}'.format(response.json().get('request'), errors.get(request_type).get(response.json().get('request'))))
# ------------------------------------------------------------------------------- #
def reportJob(self, jobID):
if not jobID:
raise RuntimeError("2Captcha: Error bad job id to request reCaptcha.")
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get('status') == 1:
return response
self.checkErrorStatus(response, 'res.php')
return None
response = polling.poll(
lambda: self.session.get(
'{}/res.php'.format(self.host),
params={
'key': self.api_key,
'action': 'reportbad',
'id': jobID,
'json': '1'
}
),
check_success=_checkRequest,
step=5,
timeout=180
)
if response:
return True
else:
raise RuntimeError("2Captcha: Error - Failed to report bad reCaptcha solve.")
# ------------------------------------------------------------------------------- #
def requestJob(self, jobID):
if not jobID:
raise RuntimeError("2Captcha: Error bad job id to request reCaptcha.")
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get('status') == 1:
return response
self.checkErrorStatus(response, 'res.php')
return None
response = polling.poll(
lambda: self.session.get(
'{}/res.php'.format(self.host),
params={
'key': self.api_key,
'action': 'get',
'id': jobID,
'json': '1'
}
),
check_success=_checkRequest,
step=5,
timeout=180
)
if response:
return response.json().get('request')
else:
raise RuntimeError("2Captcha: Error failed to solve reCaptcha.")
# ------------------------------------------------------------------------------- #
def requestSolve(self, site_url, site_key):
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get("status") == 1 and response.json().get('request'):
return response
self.checkErrorStatus(response, 'in.php')
return None
response = polling.poll(
lambda: self.session.post(
'{}/in.php'.format(self.host),
data={
'key': self.api_key,
'method': 'userrecaptcha',
'googlekey': site_key,
'pageurl': site_url,
'json': '1',
'soft_id': '5507698'
},
allow_redirects=False
),
check_success=_checkRequest,
step=5,
timeout=180
)
if response:
return response.json().get('request')
else:
raise RuntimeError('2Captcha: Error no job id was returned.')
# ------------------------------------------------------------------------------- #
def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
jobID = None
if not reCaptchaParams.get('api_key'):
raise ValueError("2Captcha: Missing api_key parameter.")
self.api_key = reCaptchaParams.get('api_key')
if reCaptchaParams.get('proxy'):
self.session.proxies = reCaptchaParams.get('proxies')
try:
jobID = self.requestSolve(site_url, site_key)
return self.requestJob(jobID)
except polling.TimeoutException:
try:
if jobID:
self.reportJob(jobID)
except polling.TimeoutException:
raise RuntimeError("2Captcha: reCaptcha solve took to long and also failed reporting the job.")
raise RuntimeError("2Captcha: reCaptcha solve took to long to execute, aborting.")
# ------------------------------------------------------------------------------- #
captchaSolver()
lib/cloudscraper/reCaptcha/__init__.py
+12 -10
View File
@@ -1,3 +1,4 @@
import re
import sys import sys
import logging import logging
import abc import abc
@@ -7,20 +8,20 @@ if sys.version_info >= (3, 4):
else: else:
ABC = abc.ABCMeta('ABC', (), {}) ABC = abc.ABCMeta('ABC', (), {})
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
BUG_REPORT = 'Cloudflare may have changed their technique, or there may be a bug in the script.'
##########################################################################################################################################################
captchaSolvers = {} captchaSolvers = {}
# ------------------------------------------------------------------------------- #
class reCaptcha(ABC): class reCaptcha(ABC):
@abc.abstractmethod @abc.abstractmethod
def __init__(self, name): def __init__(self, name):
captchaSolvers[name] = self captchaSolvers[name] = self
# ------------------------------------------------------------------------------- #
@classmethod @classmethod
def dynamicImport(cls, name): def dynamicImport(cls, name):
if name not in captchaSolvers: if name not in captchaSolvers:
@@ -34,13 +35,14 @@ class reCaptcha(ABC):
return captchaSolvers[name] return captchaSolvers[name]
# ------------------------------------------------------------------------------- #
@abc.abstractmethod @abc.abstractmethod
def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams): def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
pass pass
# ------------------------------------------------------------------------------- # def solveCaptcha(self, ret, reCaptchaParams):
try:
site_key = re.search('data-sitekey="(.+?)"', ret.text).group(1)
except Exception as e:
raise ValueError("Unable to parse Cloudflare\'s reCaptcha variable 'data-sitekey': {} {}".format(e.message, BUG_REPORT))
def solveCaptcha(self, site_url, site_key, reCaptchaParams): return self.getCaptchaAnswer(ret.url, site_key, reCaptchaParams)
return self.getCaptchaAnswer(site_url, site_key, reCaptchaParams)
lib/cloudscraper/reCaptcha/anticaptcha.py
+14 -10
View File
@@ -3,10 +3,10 @@ from __future__ import absolute_import
import sys import sys
try: try:
from python_anticaptcha import AnticaptchaClient, NoCaptchaTaskProxylessTask from python_anticaptcha import AnticaptchaClient, NoCaptchaTaskProxylessTask, NoCaptchaTask, Proxy
except ImportError: except ImportError:
sys.tracebacklimit = 0 sys.tracebacklimit = 0
raise RuntimeError("Please install the python module 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha") raise RuntimeError("Please install the python module 'python_anticaptcha' via pip or download it https://github.com/ad-m/python-anticaptcha")
from . import reCaptcha from . import reCaptcha
@@ -22,16 +22,20 @@ class captchaSolver(reCaptcha):
client = AnticaptchaClient(reCaptchaParams.get('api_key')) client = AnticaptchaClient(reCaptchaParams.get('api_key'))
if reCaptchaParams.get('proxy'): if reCaptchaParams.get('proxy', False) and reCaptchaParams.get('proxies'):
client.session.proxies = reCaptchaParams.get('proxies') client.session.proxies = reCaptchaParams.get('proxies')
task = NoCaptchaTask(
site_url,
site_key,
proxy=Proxy.parse_url(
reCaptchaParams.get('proxies').get('https')
)
)
else:
task = NoCaptchaTaskProxylessTask(site_url, site_key)
task = NoCaptchaTaskProxylessTask(site_url, site_key) job = client.createTask(task)
job.join()
if not hasattr(client, 'createTaskSmee'):
sys.tracebacklimit = 0
raise RuntimeError("Please upgrade 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha")
job = client.createTaskSmee(task)
return job.get_solution_response() return job.get_solution_response()
lib/cloudscraper/reCaptcha/deathbycaptcha.py
+7 -10
View File
@@ -20,10 +20,9 @@ class captchaSolver(reCaptcha):
self.host = 'http://api.dbcapi.me/api' self.host = 'http://api.dbcapi.me/api'
self.session = requests.Session() self.session = requests.Session()
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
@staticmethod def checkErrorStatus(self, response):
def checkErrorStatus(response):
errors = dict( errors = dict(
[ [
(400, "DeathByCaptcha: 400 Bad Request"), (400, "DeathByCaptcha: 400 Bad Request"),
@@ -36,7 +35,7 @@ class captchaSolver(reCaptcha):
if response.status_code in errors: if response.status_code in errors:
raise RuntimeError(errors.get(response.status_code)) raise RuntimeError(errors.get(response.status_code))
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def login(self, username, password): def login(self, username, password):
self.username = username self.username = username
@@ -72,7 +71,7 @@ class captchaSolver(reCaptcha):
self.debugRequest(response) self.debugRequest(response)
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def reportJob(self, jobID): def reportJob(self, jobID):
if not jobID: if not jobID:
@@ -105,7 +104,7 @@ class captchaSolver(reCaptcha):
else: else:
raise RuntimeError("DeathByCaptcha: Error report failed reCaptcha.") raise RuntimeError("DeathByCaptcha: Error report failed reCaptcha.")
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def requestJob(self, jobID): def requestJob(self, jobID):
if not jobID: if not jobID:
@@ -134,7 +133,7 @@ class captchaSolver(reCaptcha):
else: else:
raise RuntimeError("DeathByCaptcha: Error failed to solve reCaptcha.") raise RuntimeError("DeathByCaptcha: Error failed to solve reCaptcha.")
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def requestSolve(self, site_url, site_key): def requestSolve(self, site_url, site_key):
def _checkRequest(response): def _checkRequest(response):
@@ -170,7 +169,7 @@ class captchaSolver(reCaptcha):
else: else:
raise RuntimeError('DeathByCaptcha: Error no job id was returned.') raise RuntimeError('DeathByCaptcha: Error no job id was returned.')
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams): def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
jobID = None jobID = None
@@ -196,6 +195,4 @@ class captchaSolver(reCaptcha):
raise RuntimeError("DeathByCaptcha: reCaptcha solve took to long to execute, aborting.") raise RuntimeError("DeathByCaptcha: reCaptcha solve took to long to execute, aborting.")
# ------------------------------------------------------------------------------- #
captchaSolver() captchaSolver()
lib/cloudscraper/user_agent/__init__.py
+17 -53
View File
@@ -1,82 +1,46 @@
import json
import os import os
import json
import random import random
import sys import logging
from collections import OrderedDict from collections import OrderedDict
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
class User_Agent(): class User_Agent():
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.headers = None self.headers = None
self.cipherSuite = []
self.loadUserAgent(*args, **kwargs) self.loadUserAgent(*args, **kwargs)
# ------------------------------------------------------------------------------- # ##########################################################################################################################################################
def loadHeaders(self, user_agents, user_agent_version):
if user_agents.get(self.browser).get('releases').get(user_agent_version).get('headers'):
self.headers = user_agents.get(self.browser).get('releases').get(user_agent_version).get('headers')
else:
self.headers = user_agents.get(self.browser).get('default_headers')
# ------------------------------------------------------------------------------- #
def filterAgents(self, releases):
filtered = {}
for release in releases:
if self.mobile and releases[release]['User-Agent']['mobile']:
filtered[release] = filtered.get(release, []) + releases[release]['User-Agent']['mobile']
if self.desktop and releases[release]['User-Agent']['desktop']:
filtered[release] = filtered.get(release, []) + releases[release]['User-Agent']['desktop']
return filtered
# ------------------------------------------------------------------------------- #
def loadUserAgent(self, *args, **kwargs): def loadUserAgent(self, *args, **kwargs):
self.browser = kwargs.pop('browser', None) browser = kwargs.pop('browser', None)
if isinstance(self.browser, dict):
self.desktop = self.browser.get('desktop', True)
self.mobile = self.browser.get('mobile', True)
self.browser = self.browser.get('browser', None)
else:
self.desktop = kwargs.pop('desktop', True)
self.mobile = kwargs.pop('mobile', True)
if not self.desktop and not self.mobile:
sys.tracebacklimit = 0
raise RuntimeError("Sorry you can't have mobile and desktop disabled at the same time.")
user_agents = json.load( user_agents = json.load(
open(os.path.join(os.path.dirname(__file__), 'browsers.json'), 'r'), open(os.path.join(os.path.dirname(__file__), 'browsers.json'), 'r'),
object_pairs_hook=OrderedDict object_pairs_hook=OrderedDict
) )
if self.browser and not user_agents.get(self.browser): if browser and not user_agents.get(browser):
sys.tracebacklimit = 0 logging.error('Sorry "{}" browser User-Agent was not found.'.format(browser))
raise RuntimeError('Sorry "{}" browser User-Agent was not found.'.format(self.browser)) raise
if not self.browser: if not browser:
self.browser = random.SystemRandom().choice(list(user_agents)) browser = random.SystemRandom().choice(list(user_agents))
self.cipherSuite = user_agents.get(self.browser).get('cipherSuite', []) user_agent_version = random.SystemRandom().choice(list(user_agents.get(browser).get('releases')))
filteredAgents = self.filterAgents(user_agents.get(self.browser).get('releases')) if user_agents.get(browser).get('releases').get(user_agent_version).get('headers'):
self.headers = user_agents.get(browser).get('releases').get(user_agent_version).get('headers')
else:
self.headers = user_agents.get(browser).get('default_headers')
user_agent_version = random.SystemRandom().choice(list(filteredAgents)) self.headers['User-Agent'] = random.SystemRandom().choice(user_agents.get(browser).get('releases').get(user_agent_version).get('User-Agent'))
self.loadHeaders(user_agents, user_agent_version)
self.headers['User-Agent'] = random.SystemRandom().choice(filteredAgents[user_agent_version])
if not kwargs.get('allow_brotli', False): if not kwargs.get('allow_brotli', False):
if 'br' in self.headers['Accept-Encoding']: if 'br' in self.headers['Accept-Encoding']:
lib/cloudscraper/user_agent/browsers.json
+11090 -13144
View File
File diff suppressed because it is too large Load Diff
lib/js2py/base.py
+10 -3
View File
@@ -5,7 +5,6 @@ import re
from .translators.friendly_nodes import REGEXP_CONVERTER from .translators.friendly_nodes import REGEXP_CONVERTER
from .utils.injector import fix_js_args from .utils.injector import fix_js_args
from types import FunctionType, ModuleType, GeneratorType, BuiltinFunctionType, MethodType, BuiltinMethodType from types import FunctionType, ModuleType, GeneratorType, BuiltinFunctionType, MethodType, BuiltinMethodType
from math import floor, log10
import traceback import traceback
try: try:
import numpy import numpy
@@ -604,7 +603,15 @@ class PyJs(object):
elif typ == 'Boolean': elif typ == 'Boolean':
return Js('true') if self.value else Js('false') return Js('true') if self.value else Js('false')
elif typ == 'Number': #or self.Class=='Number': elif typ == 'Number': #or self.Class=='Number':
return Js(unicode(js_dtoa(self.value))) if self.is_nan():
return Js('NaN')
elif self.is_infinity():
sign = '-' if self.value < 0 else ''
return Js(sign + 'Infinity')
elif isinstance(self.value,
long) or self.value.is_integer(): # dont print .0
return Js(unicode(int(self.value)))
return Js(unicode(self.value)) # accurate enough
elif typ == 'String': elif typ == 'String':
return self return self
else: #object else: #object
@@ -1039,7 +1046,7 @@ def PyJsComma(a, b):
return b return b
from .internals.simplex import JsException as PyJsException, js_dtoa from .internals.simplex import JsException as PyJsException
import pyjsparser import pyjsparser
pyjsparser.parser.ENABLE_JS2PY_ERRORS = lambda msg: MakeError('SyntaxError', msg) pyjsparser.parser.ENABLE_JS2PY_ERRORS = lambda msg: MakeError('SyntaxError', msg)
lib/js2py/constructors/time_helpers.py
+1 -1
View File
@@ -4,7 +4,7 @@ import datetime
import warnings import warnings
try: try:
from lib.tzlocal import get_localzone from tzlocal import get_localzone
LOCAL_ZONE = get_localzone() LOCAL_ZONE = get_localzone()
except: # except all problems... except: # except all problems...
warnings.warn( warnings.warn(
lib/js2py/evaljs.py
+8 -24
View File
@@ -116,52 +116,36 @@ def eval_js(js):
def eval_js6(js): def eval_js6(js):
"""Just like eval_js but with experimental support for js6 via babel."""
return eval_js(js6_to_js5(js)) return eval_js(js6_to_js5(js))
def translate_js6(js): def translate_js6(js):
"""Just like translate_js but with experimental support for js6 via babel."""
return translate_js(js6_to_js5(js)) return translate_js(js6_to_js5(js))
class EvalJs(object): class EvalJs(object):
"""This class supports continuous execution of javascript under same context. """This class supports continuous execution of javascript under same context.
>>> ctx = EvalJs() >>> js = EvalJs()
>>> ctx.execute('var a = 10;function f(x) {return x*x};') >>> js.execute('var a = 10;function f(x) {return x*x};')
>>> ctx.f(9) >>> js.f(9)
81 81
>>> ctx.a >>> js.a
10 10
context is a python dict or object that contains python variables that should be available to JavaScript context is a python dict or object that contains python variables that should be available to JavaScript
For example: For example:
>>> ctx = EvalJs({'a': 30}) >>> js = EvalJs({'a': 30})
>>> ctx.execute('var x = a') >>> js.execute('var x = a')
>>> ctx.x >>> js.x
30 30
You can enable JS require function via enable_require. With this feature enabled you can use js modules
from npm, for example:
>>> ctx = EvalJs(enable_require=True)
>>> ctx.execute("var esprima = require('esprima');")
>>> ctx.execute("esprima.parse('var a = 1')")
You can run interactive javascript console with console method!""" You can run interactive javascript console with console method!"""
def __init__(self, context={}, enable_require=False): def __init__(self, context={}):
self.__dict__['_context'] = {} self.__dict__['_context'] = {}
exec (DEFAULT_HEADER, self._context) exec (DEFAULT_HEADER, self._context)
self.__dict__['_var'] = self._context['var'].to_python() self.__dict__['_var'] = self._context['var'].to_python()
if enable_require:
def _js_require_impl(npm_module_name):
from .node_import import require
from .base import to_python
return require(to_python(npm_module_name), context=self._context)
setattr(self._var, 'require', _js_require_impl)
if not isinstance(context, dict): if not isinstance(context, dict):
try: try:
context = context.__dict__ context = context.__dict__
lib/js2py/internals/constructors/jsfunction.py
+1 -2
View File
@@ -42,7 +42,6 @@ def executable_code(code_str, space, global_context=True):
space.byte_generator.emit('LABEL', skip) space.byte_generator.emit('LABEL', skip)
space.byte_generator.emit('NOP') space.byte_generator.emit('NOP')
space.byte_generator.restore_state() space.byte_generator.restore_state()
space.byte_generator.exe.compile( space.byte_generator.exe.compile(
start_loc=old_tape_len start_loc=old_tape_len
) # dont read the code from the beginning, dont be stupid! ) # dont read the code from the beginning, dont be stupid!
@@ -72,5 +71,5 @@ def _eval(this, args):
def log(this, args): def log(this, args):
print(' '.join(map(to_string, args))) print ' '.join(map(to_string, args))
return undefined return undefined
lib/js2py/internals/prototypes/jsstring.py
+1 -1
View File
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
import re import re
from ..conversions import * from ..conversions import *
from ..func_utils import * from ..func_utils import *
from .jsregexp import RegExpExec from jsregexp import RegExpExec
DIGS = set(u'0123456789') DIGS = set(u'0123456789')
WHITE = u"\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF" WHITE = u"\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF"
lib/js2py/node_import.py
+9 -41
View File
@@ -1,6 +1,6 @@
__all__ = ['require'] __all__ = ['require']
import subprocess, os, codecs, glob import subprocess, os, codecs, glob
from .evaljs import translate_js, DEFAULT_HEADER from .evaljs import translate_js
import six import six
DID_INIT = False DID_INIT = False
DIRNAME = os.path.dirname(os.path.abspath(__file__)) DIRNAME = os.path.dirname(os.path.abspath(__file__))
@@ -15,7 +15,7 @@ def _init():
'node -v', shell=True, cwd=DIRNAME 'node -v', shell=True, cwd=DIRNAME
) == 0, 'You must have node installed! run: brew install node' ) == 0, 'You must have node installed! run: brew install node'
assert subprocess.call( assert subprocess.call(
'cd %s;npm install babel-core babel-cli babel-preset-es2015 babel-polyfill babelify browserify browserify-shim' 'cd %s;npm install babel-core babel-cli babel-preset-es2015 babel-polyfill babelify browserify'
% repr(DIRNAME), % repr(DIRNAME),
shell=True, shell=True,
cwd=DIRNAME) == 0, 'Could not link required node_modules' cwd=DIRNAME) == 0, 'Could not link required node_modules'
@@ -46,18 +46,12 @@ GET_FROM_GLOBALS_FUNC = '''
''' '''
def _get_module_py_name(module_name):
return module_name.replace('-', '_')
def _get_module_var_name(module_name): def require(module_name, include_polyfill=False, update=False):
return _get_module_py_name(module_name).rpartition('/')[-1]
def _get_and_translate_npm_module(module_name, include_polyfill=False, update=False):
assert isinstance(module_name, str), 'module_name must be a string!' assert isinstance(module_name, str), 'module_name must be a string!'
py_name = _get_module_py_name(module_name) py_name = module_name.replace('-', '_')
module_filename = '%s.py' % py_name module_filename = '%s.py' % py_name
var_name = _get_module_var_name(module_name) var_name = py_name.rpartition('/')[-1]
if not os.path.exists(os.path.join(PY_NODE_MODULES_PATH, if not os.path.exists(os.path.join(PY_NODE_MODULES_PATH,
module_filename)) or update: module_filename)) or update:
_init() _init()
@@ -83,7 +77,7 @@ def _get_and_translate_npm_module(module_name, include_polyfill=False, update=Fa
# convert the module # convert the module
assert subprocess.call( assert subprocess.call(
'''node -e "(require('browserify')('./%s').bundle(function (err,data) {if (err) {console.log(err);throw new Error(err);};fs.writeFile('%s', require('babel-core').transform(data, {'presets': require('babel-preset-es2015')}).code, ()=>{});}))"''' '''node -e "(require('browserify')('./%s').bundle(function (err,data) {fs.writeFile('%s', require('babel-core').transform(data, {'presets': require('babel-preset-es2015')}).code, ()=>{});}))"'''
% (in_file_name, out_file_name), % (in_file_name, out_file_name),
shell=True, shell=True,
cwd=DIRNAME, cwd=DIRNAME,
@@ -94,8 +88,7 @@ def _get_and_translate_npm_module(module_name, include_polyfill=False, update=Fa
"utf-8") as f: "utf-8") as f:
js_code = f.read() js_code = f.read()
os.remove(os.path.join(DIRNAME, out_file_name)) os.remove(os.path.join(DIRNAME, out_file_name))
if len(js_code) < 50:
raise RuntimeError("Candidate JS bundle too short - likely browserify issue.")
js_code += GET_FROM_GLOBALS_FUNC js_code += GET_FROM_GLOBALS_FUNC
js_code += ';var %s = getFromGlobals(%s);%s' % ( js_code += ';var %s = getFromGlobals(%s);%s' % (
var_name, repr(module_name), var_name) var_name, repr(module_name), var_name)
@@ -114,32 +107,7 @@ def _get_and_translate_npm_module(module_name, include_polyfill=False, update=Fa
os.path.join(PY_NODE_MODULES_PATH, module_filename), "r", os.path.join(PY_NODE_MODULES_PATH, module_filename), "r",
"utf-8") as f: "utf-8") as f:
py_code = f.read() py_code = f.read()
return py_code
context = {}
def require(module_name, include_polyfill=False, update=False, context=None):
"""
Installs the provided npm module, exports a js bundle via browserify, converts to ECMA 5.1 via babel and
finally translates the generated JS bundle to Python via Js2Py.
Returns a pure python object that behaves like the installed module. Nice!
:param module_name: Name of the npm module to require. For example 'esprima'.
:param include_polyfill: Whether the babel-polyfill should be included as part of the translation. May be needed
for some modules that use unsupported features.
:param update: Whether to force update the translation. Otherwise uses a cached version if exists.
:param context: Optional context in which the translated module should be executed in. If provided, the
header (js2py imports) will be skipped as it is assumed that the context already has all the necessary imports.
:return: The JsObjectWrapper containing the translated module object. Can be used like a standard python object.
"""
py_code = _get_and_translate_npm_module(module_name, include_polyfill=include_polyfill, update=update)
# this is a bit hacky but we need to strip the default header from the generated code...
if context is not None:
if not py_code.startswith(DEFAULT_HEADER):
# new header version? retranslate...
assert not update, "Unexpected header."
py_code = _get_and_translate_npm_module(module_name, include_polyfill=include_polyfill, update=True)
assert py_code.startswith(DEFAULT_HEADER), "Unexpected header."
py_code = py_code[len(DEFAULT_HEADER):]
context = {} if context is None else context
exec (py_code, context) exec (py_code, context)
return context['var'][_get_module_var_name(module_name)].to_py() return context['var'][var_name].to_py()
lib/js2py/prototypes/jsfunction.py
+3 -2
View File
@@ -6,6 +6,8 @@ if six.PY3:
xrange = range xrange = range
unicode = str unicode = str
# todo fix apply and bind
class FunctionPrototype: class FunctionPrototype:
def toString(): def toString():
@@ -39,7 +41,6 @@ class FunctionPrototype:
return this.call(obj, args) return this.call(obj, args)
def bind(thisArg): def bind(thisArg):
arguments_ = arguments
target = this target = this
if not target.is_callable(): if not target.is_callable():
raise this.MakeError( raise this.MakeError(
@@ -47,5 +48,5 @@ class FunctionPrototype:
if len(arguments) <= 1: if len(arguments) <= 1:
args = () args = ()
else: else:
args = tuple([arguments_[e] for e in xrange(1, len(arguments_))]) args = tuple([arguments[e] for e in xrange(1, len(arguments))])
return this.PyJsBoundFunction(target, thisArg, args) return this.PyJsBoundFunction(target, thisArg, args)
lib/js2py/translators/translating_nodes.py
+55 -64
View File
@@ -155,7 +155,7 @@ def limited(func):
inf = float('inf') inf = float('inf')
def Literal(type, value, raw, regex=None): def Literal(type, value, raw, regex=None, comments=None):
if regex: # regex if regex: # regex
return 'JsRegExp(%s)' % repr(compose_regex(value)) return 'JsRegExp(%s)' % repr(compose_regex(value))
elif value is None: # null elif value is None: # null
@@ -165,12 +165,12 @@ def Literal(type, value, raw, regex=None):
return 'Js(%s)' % repr(value) if value != inf else 'Js(float("inf"))' return 'Js(%s)' % repr(value) if value != inf else 'Js(float("inf"))'
def Identifier(type, name): def Identifier(type, name, comments=None):
return 'var.get(%s)' % repr(name) return 'var.get(%s)' % repr(name)
@limited @limited
def MemberExpression(type, computed, object, property): def MemberExpression(type, computed, object, property, comments=None):
far_left = trans(object) far_left = trans(object)
if computed: # obj[prop] type accessor if computed: # obj[prop] type accessor
# may be literal which is the same in every case so we can save some time on conversion # may be literal which is the same in every case so we can save some time on conversion
@@ -183,12 +183,12 @@ def MemberExpression(type, computed, object, property):
return far_left + '.get(%s)' % prop return far_left + '.get(%s)' % prop
def ThisExpression(type): def ThisExpression(type, comments=None):
return 'var.get(u"this")' return 'var.get(u"this")'
@limited @limited
def CallExpression(type, callee, arguments): def CallExpression(type, callee, arguments, comments=None):
arguments = [trans(e) for e in arguments] arguments = [trans(e) for e in arguments]
if callee['type'] == 'MemberExpression': if callee['type'] == 'MemberExpression':
far_left = trans(callee['object']) far_left = trans(callee['object'])
@@ -210,47 +210,38 @@ def CallExpression(type, callee, arguments):
# ========== ARRAYS ============ # ========== ARRAYS ============
def ArrayExpression(type, elements): # todo fix null inside problem def ArrayExpression(type, elements, comments=None): # todo fix null inside problem
return 'Js([%s])' % ', '.join(trans(e) if e else 'None' for e in elements) return 'Js([%s])' % ', '.join(trans(e) if e else 'None' for e in elements)
# ========== OBJECTS ============= # ========== OBJECTS =============
def ObjectExpression(type, properties): def ObjectExpression(type, properties, comments=None):
name = None name = inline_stack.require('Object')
elems = [] elems = []
after = '' after = ''
for p in properties: for p in properties:
if p['kind'] == 'init': if p['kind'] == 'init':
elems.append('%s:%s' % Property(**p)) elems.append('%s:%s' % Property(**p))
elif p['kind'] == 'set':
k, setter = Property(
**p
) # setter is just a lval referring to that function, it will be defined in InlineStack automatically
after += '%s.define_own_property(%s, {"set":%s, "configurable":True, "enumerable":True})\n' % (
name, k, setter)
elif p['kind'] == 'get':
k, getter = Property(**p)
after += '%s.define_own_property(%s, {"get":%s, "configurable":True, "enumerable":True})\n' % (
name, k, getter)
else: else:
if name is None: raise RuntimeError('Unexpected object propery kind')
name = inline_stack.require('Object') obj = '%s = Js({%s})\n' % (name, ','.join(elems))
if p['kind'] == 'set': inline_stack.define(name, obj + after)
k, setter = Property( return name
**p
) # setter is just a lval referring to that function, it will be defined in InlineStack automatically
after += '%s.define_own_property(%s, {"set":%s, "configurable":True, "enumerable":True})\n' % (
name, k, setter)
elif p['kind'] == 'get':
k, getter = Property(**p)
after += '%s.define_own_property(%s, {"get":%s, "configurable":True, "enumerable":True})\n' % (
name, k, getter)
else:
raise RuntimeError('Unexpected object propery kind')
definition = 'Js({%s})' % ','.join(elems)
if name is None:
return definition
body = '%s = %s\n' % (name, definition)
body += after
body += 'return %s\n' % name
code = 'def %s():\n%s' % (name, indent(body))
inline_stack.define(name, code)
return name + '()'
def Property(type, kind, key, computed, value, method, shorthand): def Property(type, kind, key, computed, value, method, shorthand, comments=None):
if shorthand or computed: if shorthand or computed:
raise NotImplementedError( raise NotImplementedError(
'Shorthand and Computed properties not implemented!') 'Shorthand and Computed properties not implemented!')
@@ -265,7 +256,7 @@ def Property(type, kind, key, computed, value, method, shorthand):
@limited @limited
def UnaryExpression(type, operator, argument, prefix): def UnaryExpression(type, operator, argument, prefix, comments=None):
a = trans( a = trans(
argument, standard=True argument, standard=True
) # unary involve some complex operations so we cant use line shorteners here ) # unary involve some complex operations so we cant use line shorteners here
@@ -280,7 +271,7 @@ def UnaryExpression(type, operator, argument, prefix):
@limited @limited
def BinaryExpression(type, operator, left, right): def BinaryExpression(type, operator, left, right, comments=None):
a = trans(left) a = trans(left)
b = trans(right) b = trans(right)
# delegate to our friends # delegate to our friends
@@ -288,7 +279,7 @@ def BinaryExpression(type, operator, left, right):
@limited @limited
def UpdateExpression(type, operator, argument, prefix): def UpdateExpression(type, operator, argument, prefix, comments=None):
a = trans( a = trans(
argument, standard=True argument, standard=True
) # also complex operation involving parsing of the result so no line length reducing here ) # also complex operation involving parsing of the result so no line length reducing here
@@ -296,7 +287,7 @@ def UpdateExpression(type, operator, argument, prefix):
@limited @limited
def AssignmentExpression(type, operator, left, right): def AssignmentExpression(type, operator, left, right, comments=None):
operator = operator[:-1] operator = operator[:-1]
if left['type'] == 'Identifier': if left['type'] == 'Identifier':
if operator: if operator:
@@ -328,12 +319,12 @@ six
@limited @limited
def SequenceExpression(type, expressions): def SequenceExpression(type, expressions, comments=None):
return reduce(js_comma, (trans(e) for e in expressions)) return reduce(js_comma, (trans(e) for e in expressions))
@limited @limited
def NewExpression(type, callee, arguments): def NewExpression(type, callee, arguments, comments=None):
return trans(callee) + '.create(%s)' % ', '.join( return trans(callee) + '.create(%s)' % ', '.join(
trans(e) for e in arguments) trans(e) for e in arguments)
@@ -341,7 +332,7 @@ def NewExpression(type, callee, arguments):
@limited @limited
def ConditionalExpression( def ConditionalExpression(
type, test, consequent, type, test, consequent,
alternate): # caused plenty of problems in my home-made translator :) alternate, comments=None): # caused plenty of problems in my home-made translator :)
return '(%s if %s else %s)' % (trans(consequent), trans(test), return '(%s if %s else %s)' % (trans(consequent), trans(test),
trans(alternate)) trans(alternate))
@@ -349,49 +340,49 @@ def ConditionalExpression(
# =========== STATEMENTS ============= # =========== STATEMENTS =============
def BlockStatement(type, body): def BlockStatement(type, body, comments=None):
return StatementList( return StatementList(
body) # never returns empty string! In the worst case returns pass\n body) # never returns empty string! In the worst case returns pass\n
def ExpressionStatement(type, expression): def ExpressionStatement(type, expression, comments=None):
return trans(expression) + '\n' # end expression space with new line return trans(expression) + '\n' # end expression space with new line
def BreakStatement(type, label): def BreakStatement(type, label, comments=None):
if label: if label:
return 'raise %s("Breaked")\n' % (get_break_label(label['name'])) return 'raise %s("Breaked")\n' % (get_break_label(label['name']))
else: else:
return 'break\n' return 'break\n'
def ContinueStatement(type, label): def ContinueStatement(type, label, comments=None):
if label: if label:
return 'raise %s("Continued")\n' % (get_continue_label(label['name'])) return 'raise %s("Continued")\n' % (get_continue_label(label['name']))
else: else:
return 'continue\n' return 'continue\n'
def ReturnStatement(type, argument): def ReturnStatement(type, argument, comments=None):
return 'return %s\n' % (trans(argument) return 'return %s\n' % (trans(argument)
if argument else "var.get('undefined')") if argument else "var.get('undefined')")
def EmptyStatement(type): def EmptyStatement(type, comments=None):
return 'pass\n' return 'pass\n'
def DebuggerStatement(type): def DebuggerStatement(type, comments=None):
return 'pass\n' return 'pass\n'
def DoWhileStatement(type, body, test): def DoWhileStatement(type, body, test, comments=None):
inside = trans(body) + 'if not %s:\n' % trans(test) + indent('break\n') inside = trans(body) + 'if not %s:\n' % trans(test) + indent('break\n')
result = 'while 1:\n' + indent(inside) result = 'while 1:\n' + indent(inside)
return result return result
def ForStatement(type, init, test, update, body): def ForStatement(type, init, test, update, body, comments=None):
update = indent(trans(update)) if update else '' update = indent(trans(update)) if update else ''
init = trans(init) if init else '' init = trans(init) if init else ''
if not init.endswith('\n'): if not init.endswith('\n'):
@@ -407,7 +398,7 @@ def ForStatement(type, init, test, update, body):
return result return result
def ForInStatement(type, left, right, body, each): def ForInStatement(type, left, right, body, each, comments=None):
res = 'for PyJsTemp in %s:\n' % trans(right) res = 'for PyJsTemp in %s:\n' % trans(right)
if left['type'] == "VariableDeclaration": if left['type'] == "VariableDeclaration":
addon = trans(left) # make sure variable is registered addon = trans(left) # make sure variable is registered
@@ -426,7 +417,7 @@ def ForInStatement(type, left, right, body, each):
return res return res
def IfStatement(type, test, consequent, alternate): def IfStatement(type, test, consequent, alternate, comments=None):
# NOTE we cannot do elif because function definition inside elif statement would not be possible! # NOTE we cannot do elif because function definition inside elif statement would not be possible!
IF = 'if %s:\n' % trans(test) IF = 'if %s:\n' % trans(test)
IF += indent(trans(consequent)) IF += indent(trans(consequent))
@@ -436,7 +427,7 @@ def IfStatement(type, test, consequent, alternate):
return IF + ELSE return IF + ELSE
def LabeledStatement(type, label, body): def LabeledStatement(type, label, body, comments=None):
# todo consider using smarter approach! # todo consider using smarter approach!
inside = trans(body) inside = trans(body)
defs = '' defs = ''
@@ -457,7 +448,7 @@ def LabeledStatement(type, label, body):
return defs + inside return defs + inside
def StatementList(lis): def StatementList(lis, comments=None):
if lis: # ensure we don't return empty string because it may ruin indentation! if lis: # ensure we don't return empty string because it may ruin indentation!
code = ''.join(trans(e) for e in lis) code = ''.join(trans(e) for e in lis)
return code if code else 'pass\n' return code if code else 'pass\n'
@@ -465,7 +456,7 @@ def StatementList(lis):
return 'pass\n' return 'pass\n'
def PyimportStatement(type, imp): def PyimportStatement(type, imp, comments=None):
lib = imp['name'] lib = imp['name']
jlib = 'PyImport_%s' % lib jlib = 'PyImport_%s' % lib
code = 'import %s as %s\n' % (lib, jlib) code = 'import %s as %s\n' % (lib, jlib)
@@ -480,7 +471,7 @@ def PyimportStatement(type, imp):
return code return code
def SwitchStatement(type, discriminant, cases): def SwitchStatement(type, discriminant, cases, comments=None):
#TODO there will be a problem with continue in a switch statement.... FIX IT #TODO there will be a problem with continue in a switch statement.... FIX IT
code = 'while 1:\n' + indent('SWITCHED = False\nCONDITION = (%s)\n') code = 'while 1:\n' + indent('SWITCHED = False\nCONDITION = (%s)\n')
code = code % trans(discriminant) code = code % trans(discriminant)
@@ -500,12 +491,12 @@ def SwitchStatement(type, discriminant, cases):
return code return code
def ThrowStatement(type, argument): def ThrowStatement(type, argument, comments=None):
return 'PyJsTempException = JsToPyException(%s)\nraise PyJsTempException\n' % trans( return 'PyJsTempException = JsToPyException(%s)\nraise PyJsTempException\n' % trans(
argument) argument)
def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer): def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer, comments=None):
result = 'try:\n%s' % indent(trans(block)) result = 'try:\n%s' % indent(trans(block))
# complicated catch statement... # complicated catch statement...
if handler: if handler:
@@ -525,13 +516,13 @@ def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer):
return result return result
def LexicalDeclaration(type, declarations, kind): def LexicalDeclaration(type, declarations, kind, comments=None):
raise NotImplementedError( raise NotImplementedError(
'let and const not implemented yet but they will be soon! Check github for updates.' 'let and const not implemented yet but they will be soon! Check github for updates.'
) )
def VariableDeclarator(type, id, init): def VariableDeclarator(type, id, init, comments=None):
name = id['name'] name = id['name']
# register the name if not already registered # register the name if not already registered
Context.register(name) Context.register(name)
@@ -540,21 +531,21 @@ def VariableDeclarator(type, id, init):
return '' return ''
def VariableDeclaration(type, declarations, kind): def VariableDeclaration(type, declarations, kind, comments=None):
code = ''.join(trans(d) for d in declarations) code = ''.join(trans(d) for d in declarations)
return code if code else 'pass\n' return code if code else 'pass\n'
def WhileStatement(type, test, body): def WhileStatement(type, test, body, comments=None):
result = 'while %s:\n' % trans(test) + indent(trans(body)) result = 'while %s:\n' % trans(test) + indent(trans(body))
return result return result
def WithStatement(type, object, body): def WithStatement(type, object, body, comments=None):
raise NotImplementedError('With statement not implemented!') raise NotImplementedError('With statement not implemented!')
def Program(type, body): def Program(type, body, comments=None):
inline_stack.reset() inline_stack.reset()
code = ''.join(trans(e) for e in body) code = ''.join(trans(e) for e in body)
# here add hoisted elements (register variables and define functions) # here add hoisted elements (register variables and define functions)
@@ -568,7 +559,7 @@ def Program(type, body):
def FunctionDeclaration(type, id, params, defaults, body, generator, def FunctionDeclaration(type, id, params, defaults, body, generator,
expression): expression, comments=None):
if generator: if generator:
raise NotImplementedError('Generators not supported') raise NotImplementedError('Generators not supported')
if defaults: if defaults:
@@ -619,7 +610,7 @@ def FunctionDeclaration(type, id, params, defaults, body, generator,
def FunctionExpression(type, id, params, defaults, body, generator, def FunctionExpression(type, id, params, defaults, body, generator,
expression): expression, comments=None):
if generator: if generator:
raise NotImplementedError('Generators not supported') raise NotImplementedError('Generators not supported')
if defaults: if defaults:
lib/js2py/utils/injector.py
+1 -10
View File
@@ -115,16 +115,7 @@ def append_arguments(code_obj, new_locals):
code_obj.co_freevars, code_obj.co_cellvars) code_obj.co_freevars, code_obj.co_cellvars)
# Done modifying codestring - make the code object # Done modifying codestring - make the code object
if hasattr(code_obj, "replace"): return types.CodeType(*args)
# Python 3.8+
return code_obj.replace(
co_argcount=co_argcount + new_locals_len,
co_nlocals=code_obj.co_nlocals + new_locals_len,
co_code=code,
co_names=names,
co_varnames=varnames)
else:
return types.CodeType(*args)
def instructions(code_obj): def instructions(code_obj):