stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

KoD 0.7

Browse Source 
- nuovo metodo di override DNS
- aggiunta opzione nascondi server, se usi l'autoplay
- migliorie al codice e fix vari
This commit is contained in:
marco
2020-01-08 19:19:59 +01:00
parent d1cc659707
commit b4376525de
139 changed files with 6078 additions and 8909 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
lib/doh.py Normal file
View File

@@ -0,0 +1,77 @@
# https://github.com/stamparm/python-doh
from __future__ import print_function
import json
import re
import socket
import ssl
import subprocess
import sys
PY3 = sys.version_info >= (3, 0)
if hasattr(ssl, "_create_unverified_context"):
ssl._create_default_https_context = ssl._create_unverified_context
DOH_SERVER = "1.1.1.1" # Note: to prevent potential blocking of service based on DNS name
else:
DOH_SERVER = "cloudflare-dns.com" # Alternative servers: doh.securedns.eu, doh-de.blahdns.com, doh-jp.blahdns.com
if PY3:
import urllib.request
_urlopen = urllib.request.urlopen
_Request = urllib.request.Request
else:
import urllib2
_urlopen = urllib2.urlopen
_Request = urllib2.Request
def query(name, type='A', server=DOH_SERVER, path="/dns-query", fallback=True, verbose=False):
"""
Returns domain name query results retrieved by using DNS over HTTPS protocol
# Reference: https://developers.cloudflare.com/1.1.1.1/dns-over-https/json-format/
>>> query("one.one.one.one", fallback=False)
['1.0.0.1', '1.1.1.1']
>>> query("one", "NS")
['a.nic.one.', 'b.nic.one.', 'c.nic.one.', 'd.nic.one.']
"""
retval = None
try:
req = _Request("https://%s%s?name=%s&type=%s" % (server, path, name, type), headers={"Accept": "application/dns-json"})
content = _urlopen(req).read().decode()
reply = json.loads(content)
if "Answer" in reply:
answer = json.loads(content)["Answer"]
retval = [_["data"] for _ in answer]
else:
retval = []
except Exception as ex:
if verbose:
print("Exception occurred: '%s'" % ex)
if retval is None and fallback:
if type == 'A':
try:
retval = socket.gethostbyname_ex(name)[2]
except (socket.error, IndexError):
pass
if retval is None:
process = subprocess.Popen(("nslookup", "-q=%s" % type, name), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
content = (process.communicate()[0] or "").decode().replace("\r", "")
if "\n\n" in content and "can't" not in content.lower():
answer = content.split("\n\n", 1)[-1]
retval = re.findall(r"(?m)^%s.+= ([^=,\n]+)$" % re.escape(name), answer) or re.findall(r"Address: (.+)", answer)
if not retval:
match = re.search(r"Addresses: ([\s\d.]+)", answer)
if match:
retval = re.split(r"\s+", match.group(1).strip())
if not PY3 and retval:
retval = [_.encode() for _ in retval]
return retval

315
lib/gktools.py
View File

@@ -1,315 +0,0 @@
# -*- coding: utf-8 -*-
'''
gktools son un conjunto de funciones para ayudar a resolver enlaces a videos con "protección GK".
Lo de protección gk dudo que exista, le he llamado así pq los primeros ejemplos vistos se eran gkpluginsphp y gkpedia.
Características "GK" :
- Utiliza una cookie __cfduid
- Calcula un token criptográfico en función de un texto y una clave
- El texto se saca del html (por ejemplo de meta name="google-site-verification", pero puede ser más complejo)
- La clave para encriptar se calcula en js ofuscados que carga el html
- Se llama a otra url con una serie de parámetros, como el token, y de allí se obtienen los videos finales.
Howto:
1- descargar página
2- extraer datos y calcular los necesarios
3- descargar segunda página con el token calculado
4- extraer videos
El paso 2 es con diferencia el más variable y depende mucho de cada web/servidor!
Desofuscando los js se pueden ver los datos propios que necesita cada uno
(el texto a encriptar, la clave a usar, la url dónde hay que llamar y los parámetros)
Ver ejemplos en el código de los canales animeyt y pelispedia
Created for Alfa-addon by Alfa Developers Team 2018
'''
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
import os, base64, json, hashlib, urlparse
from core import httptools
from core import scrapertools
from platformcode import logger
from aadecode import decode as aadecode
# Descarga página y captura la petición de cookie
# -----------------------------------------------
def get_data_and_cookie(item, ck_name='__cfduid'):
headers = {'Referer': item.referer}
resp = httptools.downloadpage(item.url, headers=headers, cookies=False)
# ~ with open('gk_play1.html', 'w') as f: f.write(resp.data); f.close()
ck_value = ''
if ck_name != '':
for h in resp.headers:
ck = scrapertools.find_single_match(resp.headers[h], '%s=([^;]*)' % ck_name)
if ck:
ck_value = ck
break
return resp.data, ck_value
# Descarga página usando una cookie concreta
# ------------------------------------------
def get_data_with_cookie(url, ck_value='', referer='', ck_name='__cfduid'):
headers = {'Cookie': ck_name+'='+ck_value}
if referer != '': headers['referer'] = referer
data = httptools.downloadpage(url, headers=headers, cookies=False).data
# ~ with open('gk_play2.html', 'w') as f: f.write(data); f.close()
return data
# Descarga json usando una cookie concreta
# ----------------------------------------
def get_data_json(url, post, ck_value='', referer='', ck_name='__cfduid'):
headers = {'Content-Type': 'application/x-www-form-urlencoded', 'Cookie': ck_name+'='+ck_value}
if referer != '': headers['referer'] = referer
data = httptools.downloadpage(url, post=post, headers=headers, cookies=False).data
# ~ with open('gk_play3.html', 'w') as f: f.write(data); f.close()
return data
# Obtiene link de una llamada javascript Play() o de la url
# ---------------------------------------------------------
def get_play_link_id(data, url):
playparms = scrapertools.find_single_match(data, 'Play\("([^"]*)","([^"]*)","([^"]*)"')
if playparms:
link = playparms[0]
subtitle = '' if playparms[1] == '' or playparms[2] == '' else playparms[2] + playparms[1] + '.srt'
else:
subtitle = ''
link = scrapertools.find_single_match(data, 'Play\("([^"]*)"')
if not link:
link = scrapertools.find_single_match(url, 'id=([^;]*)')
return link, subtitle
# Extraer enlaces a videos de datos json
# --------------------------------------
def extraer_enlaces_json(data, referer, subtitle=''):
itemlist = []
# Ejemplos:
# {"Animeyt":[{"file":"https:\/\/storage.googleapis.com\/my-project-yt-195318.appspot.com\/slow.mp4","type":"mp4","label":"1080p"}]}
# {"link":[{"link":"http:\/\/video8.narusaku.tv\/static\/720p\/2.1208982.2039540?md5=B64FKYNbFuWvxkGcSbtz2Q&expires=1528839657","label":"720p","type":"mp4"},{"link":"http:\/\/video5.narusaku.tv\/static\/480p\/2.1208982.2039540?md5=yhLG_3VghEUSd5YlCXOTBQ&expires=1528839657","label":"480p","type":"mp4","default":true},{"link":"http:\/\/video3.narusaku.tv\/static\/360p\/2.1208982.2039540?md5=vC0ZJkxRwV1rVBdeF7D4iA&expires=1528839657","label":"360p","type":"mp4"},{"link":"http:\/\/video2.narusaku.tv\/static\/240p\/2.1208982.2039540?md5=b-y_-rgrLMW7hJwFQSD8Tw&expires=1528839657","label":"240p","type":"mp4"}]}
# {"link":"https:\/\/storage.googleapis.com\/cloudflare-caching-pelispedia.appspot.com\/cache\/16050.mp4","type":"mp4"}
# {"Harbinger":[{"Harbinger":"...","type":"...","label":"..."}], ...}
data = data.replace('"Harbinger"', '"file"')
# Intentar como json
# ------------------
try:
json_data = json.loads(data)
enlaces = analizar_enlaces_json(json_data)
for enlace in enlaces:
url = enlace['link'] if 'link' in enlace else enlace['file']
if not url.startswith('http'): url = aadecode(base64.b64decode(url)) # necesario para "Harbinger"
if not url.startswith('http'): url = decode_rijndael(url) # post-"Harbinger" en algunos casos
tit = ''
if 'type' in enlace: tit += '[%s]' % enlace['type']
if 'label' in enlace: tit += '[%s]' % enlace['label']
if tit == '': tit = '.mp4'
itemlist.append([tit, corregir_url(url, referer), 0, subtitle])
# Sino, intentar como texto
# -------------------------
except:
matches = scrapertools.find_multiple_matches(data, '"link"\s*:\s*"([^"]*)"\s*,\s*"label"\s*:\s*"([^"]*)"\s*,\s*"type"\s*:\s*"([^"]*)"')
if matches:
for url, lbl, typ in matches:
itemlist.append(['[%s][%s]' % (typ, lbl), corregir_url(url, referer), 0, subtitle])
else:
url = scrapertools.find_single_match(data, '"link"\s*:\s*"([^"]*)"')
if url:
itemlist.append(['.mp4', corregir_url(url, referer), 0, subtitle])
return itemlist
# Función recursiva que busca videos en un diccionario
# ----------------------------------------------------
def analizar_enlaces_json(d):
itemlist = []
found = {}
for k, v in d.iteritems():
if k in ['file','link','type','label'] and not isinstance(v, list):
found[k] = v
if isinstance(v, list):
for l in v:
if isinstance(l, dict): itemlist += analizar_enlaces_json(l)
if 'file' in found or 'link' in found:
itemlist.append(found)
return itemlist
# Correcciones en las urls finales obtenidas
# ------------------------------------------
def corregir_url(url, referer):
url = url.replace('\/', '/')
if 'chomikuj.pl/' in url: url += "|Referer=%s" % referer
return url
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Conversion tipo hexa que hay en el js
# -------------------------------------
def toHex(txt):
ret = ''
for i in range(len(txt)):
ret += str(hex(ord(txt[i]))).replace('x','')[-2:]
return ret
# Subrutinas de encriptación
# --------------------------
def md5_dominio(url): # sutorimux/kubechi
h = hashlib.md5(urlparse.urlparse(url).netloc)
return h.hexdigest()
def transforma_gsv(gsv, valor):
llista = range(256)
a = 0
for i in range(256):
a = (a + llista[i] + ord(gsv[i % len(gsv)]) ) % 256
b = llista[i]
llista[i] = llista[a]
llista[a] = b
ret = ''
a = 0; b= 0
for i in range(len(valor)):
a = (a + 1) % 256
b = (b + llista[a]) % 256
c = llista[a]
llista[a] = llista[b]
llista[b] = c
ret += chr(ord(valor[i]) ^ llista[(llista[a] + llista[b]) % 256])
return base64.b64encode(ret)
# Codificar/Decodificar con Rijndael
# ----------------------------------
def encode_rijndael(msg, IV, key):
import rijndael
return rijndael.cbc_encrypt(msg, IV, key)
def decode_rijndael(txt, preIV='b3512f4972d314da9', key='3e1a854e7d5835ab99d99a29afec8bbb'):
import rijndael
msg = base64.b64decode(txt[:-15])
IV = preIV + txt[-15:]
deco = rijndael.cbc_decrypt(msg, IV, key)
return deco.replace(chr(0), '')
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Generar un token válido a partir de un texto y una clave
# --------------------------------------------------------
# gsv: google-site-verification, obtenido de '<meta name="google-site-verification" content="([^"]*)"'
# pwd: Password
def generar_token(gsv, pwd):
txt = obtener_cripto(pwd, gsv)
# ~ logger.info('Texto pre token %s' % txt)
_0x382d28 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
valors = [0, 0, 0]
cicle = 0
retorn = ''
for ch in txt:
valors[cicle] = ord(ch)
cicle += 1
if cicle == 3:
primer = _0x382d28[valors[0] >> 0x2]
segon = _0x382d28[((valors[0] & 0x3) << 0x4) | (valors[1] >> 0x4)]
tercer = _0x382d28[((valors[1] & 0xf) << 0x2) | (valors[2] >> 0x6)]
quart = _0x382d28[valors[2] & 0x3f]
retorn += primer + segon + tercer + quart
valors = [0, 0, 0]
cicle = 0
return retorn
def obtener_cripto(password, plaintext):
salt = os.urandom(8)
paddingLength = len(plaintext) % 16
if paddingLength == 0:
paddedPlaintext = plaintext
else:
dif = 16 - paddingLength
paddedPlaintext = plaintext + chr(dif)*dif
kdf = evpKDF(password, salt)
iv = kdf['iv']
try: # Intentar con librería AES del sistema
from Crypto.Cipher import AES
cipherSpec = AES.new(kdf['key'], AES.MODE_CBC, iv)
except: # Si falla intentar con librería del addon
import jscrypto
cipherSpec = jscrypto.new(kdf['key'], jscrypto.MODE_CBC, iv)
ciphertext = cipherSpec.encrypt(paddedPlaintext)
return json.dumps({'ct': base64.b64encode(ciphertext), 'iv': iv.encode("hex"), 's': salt.encode("hex")}, sort_keys=True, separators=(',', ':'))
def evpKDF(passwd, salt, key_size=8, iv_size=4, iterations=1, hash_algorithm="md5"):
target_key_size = key_size + iv_size
derived_bytes = ""
number_of_derived_words = 0
block = None
hasher = hashlib.new(hash_algorithm)
while number_of_derived_words < target_key_size:
if block is not None:
hasher.update(block)
hasher.update(passwd)
hasher.update(salt)
block = hasher.digest()
hasher = hashlib.new(hash_algorithm)
for i in range(1, iterations):
hasher.update(block)
block = hasher.digest()
hasher = hashlib.new(hash_algorithm)
derived_bytes += block[0: min(len(block), (target_key_size - number_of_derived_words) * 4)]
number_of_derived_words += len(block)/4
return {
"key": derived_bytes[0: key_size * 4],
"iv": derived_bytes[key_size * 4:]
}

312
lib/jjdecode.py
View File

@@ -1,312 +0,0 @@
# -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Python version of the jjdecode function written by Syed Zainudeen
# http://csc.cs.utm.my/syed/images/files/jjdecode/jjdecode.html
#
# +NCR/CRC! [ReVeRsEr] - crackinglandia@gmail.com
# Thanks to Jose Miguel Esparza (@EternalTodo) for the final push to make it work!
#
import re
class JJDecoder(object):
def __init__(self, jj_encoded_data):
self.encoded_str = jj_encoded_data
def clean(self):
return re.sub('^\s+|\s+$', '', self.encoded_str)
def checkPalindrome(self, Str):
startpos = -1
endpos = -1
gv, gvl = -1, -1
index = Str.find('"\'\\"+\'+",')
if index == 0:
startpos = Str.find('$$+"\\""+') + 8
endpos = Str.find('"\\"")())()')
gv = Str[Str.find('"\'\\"+\'+",')+9:Str.find('=~[]')]
gvl = len(gv)
else:
gv = Str[0:Str.find('=')]
gvl = len(gv)
startpos = Str.find('"\\""+') + 5
endpos = Str.find('"\\"")())()')
return (startpos, endpos, gv, gvl)
def decode(self):
self.encoded_str = self.clean()
startpos, endpos, gv, gvl = self.checkPalindrome(self.encoded_str)
if startpos == endpos:
raise Exception('No data!')
data = self.encoded_str[startpos:endpos]
b = ['___+', '__$+', '_$_+', '_$$+', '$__+', '$_$+', '$$_+', '$$$+', '$___+', '$__$+', '$_$_+', '$_$$+', '$$__+', '$$_$+', '$$$_+', '$$$$+']
str_l = '(![]+"")[' + gv + '._$_]+'
str_o = gv + '._$+'
str_t = gv + '.__+'
str_u = gv + '._+'
str_hex = gv + '.'
str_s = '"'
gvsig = gv + '.'
str_quote = '\\\\\\"'
str_slash = '\\\\\\\\'
str_lower = '\\\\"+'
str_upper = '\\\\"+' + gv + '._+'
str_end = '"+'
out = ''
while data != '':
# l o t u
if data.find(str_l) == 0:
data = data[len(str_l):]
out += 'l'
continue
elif data.find(str_o) == 0:
data = data[len(str_o):]
out += 'o'
continue
elif data.find(str_t) == 0:
data = data[len(str_t):]
out += 't'
continue
elif data.find(str_u) == 0:
data = data[len(str_u):]
out += 'u'
continue
# 0123456789abcdef
if data.find(str_hex) == 0:
data = data[len(str_hex):]
for i in range(len(b)):
if data.find(b[i]) == 0:
data = data[len(b[i]):]
out += '%x' % i
break
continue
# start of s block
if data.find(str_s) == 0:
data = data[len(str_s):]
# check if "R
if data.find(str_upper) == 0: # r4 n >= 128
data = data[len(str_upper):] # skip sig
ch_str = ''
for i in range(2): # shouldn't be more than 2 hex chars
# gv + "."+b[ c ]
if data.find(gvsig) == 0:
data = data[len(gvsig):]
for k in range(len(b)): # for every entry in b
if data.find(b[k]) == 0:
data = data[len(b[k]):]
ch_str = '%x' % k
break
else:
break
out += chr(int(ch_str, 16))
continue
elif data.find(str_lower) == 0: # r3 check if "R // n < 128
data = data[len(str_lower):] # skip sig
ch_str = ''
ch_lotux = ''
temp = ''
b_checkR1 = 0
for j in range(3): # shouldn't be more than 3 octal chars
if j > 1: # lotu check
if data.find(str_l) == 0:
data = data[len(str_l):]
ch_lotux = 'l'
break
elif data.find(str_o) == 0:
data = data[len(str_o):]
ch_lotux = 'o'
break
elif data.find(str_t) == 0:
data = data[len(str_t):]
ch_lotux = 't'
break
elif data.find(str_u) == 0:
data = data[len(str_u):]
ch_lotux = 'u'
break
# gv + "."+b[ c ]
if data.find(gvsig) == 0:
temp = data[len(gvsig):]
for k in range(8): # for every entry in b octal
if temp.find(b[k]) == 0:
if int(ch_str + str(k), 8) > 128:
b_checkR1 = 1
break
ch_str += str(k)
data = data[len(gvsig):] # skip gvsig
data = data[len(b[k]):]
break
if b_checkR1 == 1:
if data.find(str_hex) == 0: # 0123456789abcdef
data = data[len(str_hex):]
# check every element of hex decode string for a match
for i in range(len(b)):
if data.find(b[i]) == 0:
data = data[len(b[i]):]
ch_lotux = '%x' % i
break
break
else:
break
out += chr(int(ch_str,8)) + ch_lotux
continue
else: # "S ----> "SR or "S+
# if there is, loop s until R 0r +
# if there is no matching s block, throw error
match = 0;
n = None
# searching for matching pure s block
while True:
n = ord(data[0])
if data.find(str_quote) == 0:
data = data[len(str_quote):]
out += '"'
match += 1
continue
elif data.find(str_slash) == 0:
data = data[len(str_slash):]
out += '\\'
match += 1
continue
elif data.find(str_end) == 0: # reached end off S block ? +
if match == 0:
raise '+ no match S block: ' + data
data = data[len(str_end):]
break # step out of the while loop
elif data.find(str_upper) == 0: # r4 reached end off S block ? - check if "R n >= 128
if match == 0:
raise 'no match S block n>128: ' + data
data = data[len(str_upper):] # skip sig
ch_str = ''
ch_lotux = ''
for j in range(10): # shouldn't be more than 10 hex chars
if j > 1: # lotu check
if data.find(str_l) == 0:
data = data[len(str_l):]
ch_lotux = 'l'
break
elif data.find(str_o) == 0:
data = data[len(str_o):]
ch_lotux = 'o'
break
elif data.find(str_t) == 0:
data = data[len(str_t):]
ch_lotux = 't'
break
elif data.find(str_u) == 0:
data = data[len(str_u):]
ch_lotux = 'u'
break
# gv + "."+b[ c ]
if data.find(gvsig) == 0:
data = data[len(gvsig):] # skip gvsig
for k in range(len(b)): # for every entry in b
if data.find(b[k]) == 0:
data = data[len(b[k]):]
ch_str += '%x' % k
break
else:
break # done
out += chr(int(ch_str, 16))
break # step out of the while loop
elif data.find(str_lower) == 0: # r3 check if "R // n < 128
if match == 0:
raise 'no match S block n<128: ' + data
data = data[len(str_lower):] # skip sig
ch_str = ''
ch_lotux = ''
temp = ''
b_checkR1 = 0
for j in range(3): # shouldn't be more than 3 octal chars
if j > 1: # lotu check
if data.find(str_l) == 0:
data = data[len(str_l):]
ch_lotux = 'l'
break
elif data.find(str_o) == 0:
data = data[len(str_o):]
ch_lotux = 'o'
break
elif data.find(str_t) == 0:
data = data[len(str_t):]
ch_lotux = 't'
break
elif data.find(str_u) == 0:
data = data[len(str_u):]
ch_lotux = 'u'
break
# gv + "."+b[ c ]
if data.find(gvsig) == 0:
temp = data[len(gvsig):]
for k in range(8): # for every entry in b octal
if temp.find(b[k]) == 0:
if int(ch_str + str(k), 8) > 128:
b_checkR1 = 1
break
ch_str += str(k)
data = data[len(gvsig):] # skip gvsig
data = data[len(b[k]):]
break
if b_checkR1 == 1:
if data.find(str_hex) == 0: # 0123456789abcdef
data = data[len(str_hex):]
# check every element of hex decode string for a match
for i in range(len(b)):
if data.find(b[i]) == 0:
data = data[len(b[i]):]
ch_lotux = '%x' % i
break
else:
break
out += chr(int(ch_str, 8)) + ch_lotux
break # step out of the while loop
elif (0x21 <= n and n <= 0x2f) or (0x3A <= n and n <= 0x40) or ( 0x5b <= n and n <= 0x60 ) or ( 0x7b <= n and n <= 0x7f ):
out += data[0]
data = data[1:]
match += 1
continue
print 'No match : ' + data
break
return out

13
lib/js2py/base.py
View File

@@ -5,6 +5,7 @@ import re
from .translators.friendly_nodes import REGEXP_CONVERTER
from .utils.injector import fix_js_args
from types import FunctionType, ModuleType, GeneratorType, BuiltinFunctionType, MethodType, BuiltinMethodType
from math import floor, log10
import traceback
try:
import numpy
@@ -603,15 +604,7 @@ class PyJs(object):
elif typ == 'Boolean':
return Js('true') if self.value else Js('false')
elif typ == 'Number': #or self.Class=='Number':
if self.is_nan():
return Js('NaN')
elif self.is_infinity():
sign = '-' if self.value < 0 else ''
return Js(sign + 'Infinity')
elif isinstance(self.value,
long) or self.value.is_integer(): # dont print .0
return Js(unicode(int(self.value)))
return Js(unicode(self.value)) # accurate enough
return Js(unicode(js_dtoa(self.value)))
elif typ == 'String':
return self
else: #object
@@ -1046,7 +1039,7 @@ def PyJsComma(a, b):
return b
from .internals.simplex import JsException as PyJsException
from .internals.simplex import JsException as PyJsException, js_dtoa
import pyjsparser
pyjsparser.parser.ENABLE_JS2PY_ERRORS = lambda msg: MakeError('SyntaxError', msg)

32
lib/js2py/evaljs.py
View File

@@ -116,36 +116,52 @@ def eval_js(js):
def eval_js6(js):
"""Just like eval_js but with experimental support for js6 via babel."""
return eval_js(js6_to_js5(js))
def translate_js6(js):
"""Just like translate_js but with experimental support for js6 via babel."""
return translate_js(js6_to_js5(js))
class EvalJs(object):
"""This class supports continuous execution of javascript under same context.
>>> js = EvalJs()
>>> js.execute('var a = 10;function f(x) {return x*x};')
>>> js.f(9)
>>> ctx = EvalJs()
>>> ctx.execute('var a = 10;function f(x) {return x*x};')
>>> ctx.f(9)
81
>>> js.a
>>> ctx.a
10
context is a python dict or object that contains python variables that should be available to JavaScript
For example:
>>> js = EvalJs({'a': 30})
>>> js.execute('var x = a')
>>> js.x
>>> ctx = EvalJs({'a': 30})
>>> ctx.execute('var x = a')
>>> ctx.x
30
You can enable JS require function via enable_require. With this feature enabled you can use js modules
from npm, for example:
>>> ctx = EvalJs(enable_require=True)
>>> ctx.execute("var esprima = require('esprima');")
>>> ctx.execute("esprima.parse('var a = 1')")
You can run interactive javascript console with console method!"""
def __init__(self, context={}):
def __init__(self, context={}, enable_require=False):
self.__dict__['_context'] = {}
exec (DEFAULT_HEADER, self._context)
self.__dict__['_var'] = self._context['var'].to_python()
if enable_require:
def _js_require_impl(npm_module_name):
from .node_import import require
from .base import to_python
return require(to_python(npm_module_name), context=self._context)
setattr(self._var, 'require', _js_require_impl)
if not isinstance(context, dict):
try:
context = context.__dict__

5
lib/js2py/internals/constructors/jsfunction.py
View File

@@ -7,7 +7,7 @@ from ..byte_trans import ByteCodeGenerator, Code
def Function(this, args):
# convert arguments to python list of strings
a = map(to_string, tuple(args))
a = list(map(to_string, tuple(args)))
_body = u';'
_args = ()
if len(a):
@@ -42,6 +42,7 @@ def executable_code(code_str, space, global_context=True):
space.byte_generator.emit('LABEL', skip)
space.byte_generator.emit('NOP')
space.byte_generator.restore_state()
space.byte_generator.exe.compile(
start_loc=old_tape_len
) # dont read the code from the beginning, dont be stupid!
@@ -71,5 +72,5 @@ def _eval(this, args):
def log(this, args):
print ' '.join(map(to_string, args))
print(' '.join(map(to_string, args)))
return undefined

2
lib/js2py/internals/prototypes/jsstring.py
View File

@@ -4,7 +4,7 @@ from __future__ import unicode_literals
import re
from ..conversions import *
from ..func_utils import *
from jsregexp import RegExpExec
from .jsregexp import RegExpExec
DIGS = set(u'0123456789')
WHITE = u"\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF"

50
lib/js2py/node_import.py
View File

@@ -1,6 +1,6 @@
__all__ = ['require']
import subprocess, os, codecs, glob
from .evaljs import translate_js
from .evaljs import translate_js, DEFAULT_HEADER
import six
DID_INIT = False
DIRNAME = os.path.dirname(os.path.abspath(__file__))
@@ -15,7 +15,7 @@ def _init():
'node -v', shell=True, cwd=DIRNAME
) == 0, 'You must have node installed! run: brew install node'
assert subprocess.call(
'cd %s;npm install babel-core babel-cli babel-preset-es2015 babel-polyfill babelify browserify'
'cd %s;npm install babel-core babel-cli babel-preset-es2015 babel-polyfill babelify browserify browserify-shim'
% repr(DIRNAME),
shell=True,
cwd=DIRNAME) == 0, 'Could not link required node_modules'
@@ -46,12 +46,18 @@ GET_FROM_GLOBALS_FUNC = '''
'''
def _get_module_py_name(module_name):
return module_name.replace('-', '_')
def require(module_name, include_polyfill=False, update=False):
def _get_module_var_name(module_name):
return _get_module_py_name(module_name).rpartition('/')[-1]
def _get_and_translate_npm_module(module_name, include_polyfill=False, update=False):
assert isinstance(module_name, str), 'module_name must be a string!'
py_name = module_name.replace('-', '_')
py_name = _get_module_py_name(module_name)
module_filename = '%s.py' % py_name
var_name = py_name.rpartition('/')[-1]
var_name = _get_module_var_name(module_name)
if not os.path.exists(os.path.join(PY_NODE_MODULES_PATH,
module_filename)) or update:
_init()
@@ -77,7 +83,7 @@ def require(module_name, include_polyfill=False, update=False):
# convert the module
assert subprocess.call(
'''node -e "(require('browserify')('./%s').bundle(function (err,data) {fs.writeFile('%s', require('babel-core').transform(data, {'presets': require('babel-preset-es2015')}).code, ()=>{});}))"'''
'''node -e "(require('browserify')('./%s').bundle(function (err,data) {if (err) {console.log(err);throw new Error(err);};fs.writeFile('%s', require('babel-core').transform(data, {'presets': require('babel-preset-es2015')}).code, ()=>{});}))"'''
% (in_file_name, out_file_name),
shell=True,
cwd=DIRNAME,
@@ -88,7 +94,8 @@ def require(module_name, include_polyfill=False, update=False):
"utf-8") as f:
js_code = f.read()
os.remove(os.path.join(DIRNAME, out_file_name))
if len(js_code) < 50:
raise RuntimeError("Candidate JS bundle too short - likely browserify issue.")
js_code += GET_FROM_GLOBALS_FUNC
js_code += ';var %s = getFromGlobals(%s);%s' % (
var_name, repr(module_name), var_name)
@@ -107,7 +114,32 @@ def require(module_name, include_polyfill=False, update=False):
os.path.join(PY_NODE_MODULES_PATH, module_filename), "r",
"utf-8") as f:
py_code = f.read()
return py_code
context = {}
def require(module_name, include_polyfill=False, update=False, context=None):
"""
Installs the provided npm module, exports a js bundle via browserify, converts to ECMA 5.1 via babel and
finally translates the generated JS bundle to Python via Js2Py.
Returns a pure python object that behaves like the installed module. Nice!
:param module_name: Name of the npm module to require. For example 'esprima'.
:param include_polyfill: Whether the babel-polyfill should be included as part of the translation. May be needed
for some modules that use unsupported features.
:param update: Whether to force update the translation. Otherwise uses a cached version if exists.
:param context: Optional context in which the translated module should be executed in. If provided, the
header (js2py imports) will be skipped as it is assumed that the context already has all the necessary imports.
:return: The JsObjectWrapper containing the translated module object. Can be used like a standard python object.
"""
py_code = _get_and_translate_npm_module(module_name, include_polyfill=include_polyfill, update=update)
# this is a bit hacky but we need to strip the default header from the generated code...
if context is not None:
if not py_code.startswith(DEFAULT_HEADER):
# new header version? retranslate...
assert not update, "Unexpected header."
py_code = _get_and_translate_npm_module(module_name, include_polyfill=include_polyfill, update=True)
assert py_code.startswith(DEFAULT_HEADER), "Unexpected header."
py_code = py_code[len(DEFAULT_HEADER):]
context = {} if context is None else context
exec (py_code, context)
return context['var'][var_name].to_py()
return context['var'][_get_module_var_name(module_name)].to_py()

5
lib/js2py/prototypes/jsfunction.py
View File

@@ -6,8 +6,6 @@ if six.PY3:
xrange = range
unicode = str
# todo fix apply and bind
class FunctionPrototype:
def toString():
@@ -41,6 +39,7 @@ class FunctionPrototype:
return this.call(obj, args)
def bind(thisArg):
arguments_ = arguments
target = this
if not target.is_callable():
raise this.MakeError(
@@ -48,5 +47,5 @@ class FunctionPrototype:
if len(arguments) <= 1:
args = ()
else:
args = tuple([arguments[e] for e in xrange(1, len(arguments))])
args = tuple([arguments_[e] for e in xrange(1, len(arguments_))])
return this.PyJsBoundFunction(target, thisArg, args)

9
lib/js2py/test_internals.py
View File

@@ -1,9 +0,0 @@
from internals import byte_trans
from internals import seval
import pyjsparser
x = r'''
function g() {var h123 = 11; return [function g1() {return h123}, new Function('return h123')]}
g()[1]()
'''
print seval.eval_js_vm(x)

119
lib/js2py/translators/translating_nodes.py
View File

@@ -155,7 +155,7 @@ def limited(func):
inf = float('inf')
def Literal(type, value, raw, regex=None, comments=None):
def Literal(type, value, raw, regex=None):
if regex: # regex
return 'JsRegExp(%s)' % repr(compose_regex(value))
elif value is None: # null
@@ -165,12 +165,12 @@ def Literal(type, value, raw, regex=None, comments=None):
return 'Js(%s)' % repr(value) if value != inf else 'Js(float("inf"))'
def Identifier(type, name, comments=None):
def Identifier(type, name):
return 'var.get(%s)' % repr(name)
@limited
def MemberExpression(type, computed, object, property, comments=None):
def MemberExpression(type, computed, object, property):
far_left = trans(object)
if computed: # obj[prop] type accessor
# may be literal which is the same in every case so we can save some time on conversion
@@ -183,12 +183,12 @@ def MemberExpression(type, computed, object, property, comments=None):
return far_left + '.get(%s)' % prop
def ThisExpression(type, comments=None):
def ThisExpression(type):
return 'var.get(u"this")'
@limited
def CallExpression(type, callee, arguments, comments=None):
def CallExpression(type, callee, arguments):
arguments = [trans(e) for e in arguments]
if callee['type'] == 'MemberExpression':
far_left = trans(callee['object'])
@@ -210,38 +210,47 @@ def CallExpression(type, callee, arguments, comments=None):
# ========== ARRAYS ============
def ArrayExpression(type, elements, comments=None): # todo fix null inside problem
def ArrayExpression(type, elements): # todo fix null inside problem
return 'Js([%s])' % ', '.join(trans(e) if e else 'None' for e in elements)
# ========== OBJECTS =============
def ObjectExpression(type, properties, comments=None):
name = inline_stack.require('Object')
def ObjectExpression(type, properties):
name = None
elems = []
after = ''
for p in properties:
if p['kind'] == 'init':
elems.append('%s:%s' % Property(**p))
elif p['kind'] == 'set':
k, setter = Property(
**p
) # setter is just a lval referring to that function, it will be defined in InlineStack automatically
after += '%s.define_own_property(%s, {"set":%s, "configurable":True, "enumerable":True})\n' % (
name, k, setter)
elif p['kind'] == 'get':
k, getter = Property(**p)
after += '%s.define_own_property(%s, {"get":%s, "configurable":True, "enumerable":True})\n' % (
name, k, getter)
else:
raise RuntimeError('Unexpected object propery kind')
obj = '%s = Js({%s})\n' % (name, ','.join(elems))
inline_stack.define(name, obj + after)
return name
if name is None:
name = inline_stack.require('Object')
if p['kind'] == 'set':
k, setter = Property(
**p
) # setter is just a lval referring to that function, it will be defined in InlineStack automatically
after += '%s.define_own_property(%s, {"set":%s, "configurable":True, "enumerable":True})\n' % (
name, k, setter)
elif p['kind'] == 'get':
k, getter = Property(**p)
after += '%s.define_own_property(%s, {"get":%s, "configurable":True, "enumerable":True})\n' % (
name, k, getter)
else:
raise RuntimeError('Unexpected object propery kind')
definition = 'Js({%s})' % ','.join(elems)
if name is None:
return definition
body = '%s = %s\n' % (name, definition)
body += after
body += 'return %s\n' % name
code = 'def %s():\n%s' % (name, indent(body))
inline_stack.define(name, code)
return name + '()'
def Property(type, kind, key, computed, value, method, shorthand, comments=None):
def Property(type, kind, key, computed, value, method, shorthand):
if shorthand or computed:
raise NotImplementedError(
'Shorthand and Computed properties not implemented!')
@@ -256,7 +265,7 @@ def Property(type, kind, key, computed, value, method, shorthand, comments=None)
@limited
def UnaryExpression(type, operator, argument, prefix, comments=None):
def UnaryExpression(type, operator, argument, prefix):
a = trans(
argument, standard=True
) # unary involve some complex operations so we cant use line shorteners here
@@ -271,7 +280,7 @@ def UnaryExpression(type, operator, argument, prefix, comments=None):
@limited
def BinaryExpression(type, operator, left, right, comments=None):
def BinaryExpression(type, operator, left, right):
a = trans(left)
b = trans(right)
# delegate to our friends
@@ -279,7 +288,7 @@ def BinaryExpression(type, operator, left, right, comments=None):
@limited
def UpdateExpression(type, operator, argument, prefix, comments=None):
def UpdateExpression(type, operator, argument, prefix):
a = trans(
argument, standard=True
) # also complex operation involving parsing of the result so no line length reducing here
@@ -287,7 +296,7 @@ def UpdateExpression(type, operator, argument, prefix, comments=None):
@limited
def AssignmentExpression(type, operator, left, right, comments=None):
def AssignmentExpression(type, operator, left, right):
operator = operator[:-1]
if left['type'] == 'Identifier':
if operator:
@@ -319,12 +328,12 @@ six
@limited
def SequenceExpression(type, expressions, comments=None):
def SequenceExpression(type, expressions):
return reduce(js_comma, (trans(e) for e in expressions))
@limited
def NewExpression(type, callee, arguments, comments=None):
def NewExpression(type, callee, arguments):
return trans(callee) + '.create(%s)' % ', '.join(
trans(e) for e in arguments)
@@ -332,7 +341,7 @@ def NewExpression(type, callee, arguments, comments=None):
@limited
def ConditionalExpression(
type, test, consequent,
alternate, comments=None): # caused plenty of problems in my home-made translator :)
alternate): # caused plenty of problems in my home-made translator :)
return '(%s if %s else %s)' % (trans(consequent), trans(test),
trans(alternate))
@@ -340,49 +349,49 @@ def ConditionalExpression(
# =========== STATEMENTS =============
def BlockStatement(type, body, comments=None):
def BlockStatement(type, body):
return StatementList(
body) # never returns empty string! In the worst case returns pass\n
def ExpressionStatement(type, expression, comments=None):
def ExpressionStatement(type, expression):
return trans(expression) + '\n' # end expression space with new line
def BreakStatement(type, label, comments=None):
def BreakStatement(type, label):
if label:
return 'raise %s("Breaked")\n' % (get_break_label(label['name']))
else:
return 'break\n'
def ContinueStatement(type, label, comments=None):
def ContinueStatement(type, label):
if label:
return 'raise %s("Continued")\n' % (get_continue_label(label['name']))
else:
return 'continue\n'
def ReturnStatement(type, argument, comments=None):
def ReturnStatement(type, argument):
return 'return %s\n' % (trans(argument)
if argument else "var.get('undefined')")
def EmptyStatement(type, comments=None):
def EmptyStatement(type):
return 'pass\n'
def DebuggerStatement(type, comments=None):
def DebuggerStatement(type):
return 'pass\n'
def DoWhileStatement(type, body, test, comments=None):
def DoWhileStatement(type, body, test):
inside = trans(body) + 'if not %s:\n' % trans(test) + indent('break\n')
result = 'while 1:\n' + indent(inside)
return result
def ForStatement(type, init, test, update, body, comments=None):
def ForStatement(type, init, test, update, body):
update = indent(trans(update)) if update else ''
init = trans(init) if init else ''
if not init.endswith('\n'):
@@ -398,7 +407,7 @@ def ForStatement(type, init, test, update, body, comments=None):
return result
def ForInStatement(type, left, right, body, each, comments=None):
def ForInStatement(type, left, right, body, each):
res = 'for PyJsTemp in %s:\n' % trans(right)
if left['type'] == "VariableDeclaration":
addon = trans(left) # make sure variable is registered
@@ -417,7 +426,7 @@ def ForInStatement(type, left, right, body, each, comments=None):
return res
def IfStatement(type, test, consequent, alternate, comments=None):
def IfStatement(type, test, consequent, alternate):
# NOTE we cannot do elif because function definition inside elif statement would not be possible!
IF = 'if %s:\n' % trans(test)
IF += indent(trans(consequent))
@@ -427,7 +436,7 @@ def IfStatement(type, test, consequent, alternate, comments=None):
return IF + ELSE
def LabeledStatement(type, label, body, comments=None):
def LabeledStatement(type, label, body):
# todo consider using smarter approach!
inside = trans(body)
defs = ''
@@ -448,7 +457,7 @@ def LabeledStatement(type, label, body, comments=None):
return defs + inside
def StatementList(lis, comments=None):
def StatementList(lis):
if lis: # ensure we don't return empty string because it may ruin indentation!
code = ''.join(trans(e) for e in lis)
return code if code else 'pass\n'
@@ -456,7 +465,7 @@ def StatementList(lis, comments=None):
return 'pass\n'
def PyimportStatement(type, imp, comments=None):
def PyimportStatement(type, imp):
lib = imp['name']
jlib = 'PyImport_%s' % lib
code = 'import %s as %s\n' % (lib, jlib)
@@ -471,7 +480,7 @@ def PyimportStatement(type, imp, comments=None):
return code
def SwitchStatement(type, discriminant, cases, comments=None):
def SwitchStatement(type, discriminant, cases):
#TODO there will be a problem with continue in a switch statement.... FIX IT
code = 'while 1:\n' + indent('SWITCHED = False\nCONDITION = (%s)\n')
code = code % trans(discriminant)
@@ -491,12 +500,12 @@ def SwitchStatement(type, discriminant, cases, comments=None):
return code
def ThrowStatement(type, argument, comments=None):
def ThrowStatement(type, argument):
return 'PyJsTempException = JsToPyException(%s)\nraise PyJsTempException\n' % trans(
argument)
def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer, comments=None):
def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer):
result = 'try:\n%s' % indent(trans(block))
# complicated catch statement...
if handler:
@@ -516,13 +525,13 @@ def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer, com
return result
def LexicalDeclaration(type, declarations, kind, comments=None):
def LexicalDeclaration(type, declarations, kind):
raise NotImplementedError(
'let and const not implemented yet but they will be soon! Check github for updates.'
)
def VariableDeclarator(type, id, init, comments=None):
def VariableDeclarator(type, id, init):
name = id['name']
# register the name if not already registered
Context.register(name)
@@ -531,21 +540,21 @@ def VariableDeclarator(type, id, init, comments=None):
return ''
def VariableDeclaration(type, declarations, kind, comments=None):
def VariableDeclaration(type, declarations, kind):
code = ''.join(trans(d) for d in declarations)
return code if code else 'pass\n'
def WhileStatement(type, test, body, comments=None):
def WhileStatement(type, test, body):
result = 'while %s:\n' % trans(test) + indent(trans(body))
return result
def WithStatement(type, object, body, comments=None):
def WithStatement(type, object, body):
raise NotImplementedError('With statement not implemented!')
def Program(type, body, comments=None):
def Program(type, body):
inline_stack.reset()
code = ''.join(trans(e) for e in body)
# here add hoisted elements (register variables and define functions)
@@ -559,7 +568,7 @@ def Program(type, body, comments=None):
def FunctionDeclaration(type, id, params, defaults, body, generator,
expression, comments=None):
expression):
if generator:
raise NotImplementedError('Generators not supported')
if defaults:
@@ -610,7 +619,7 @@ def FunctionDeclaration(type, id, params, defaults, body, generator,
def FunctionExpression(type, id, params, defaults, body, generator,
expression, comments=None):
expression):
if generator:
raise NotImplementedError('Generators not supported')
if defaults:

11
lib/js2py/utils/injector.py
View File

@@ -115,7 +115,16 @@ def append_arguments(code_obj, new_locals):
code_obj.co_freevars, code_obj.co_cellvars)
# Done modifying codestring - make the code object
return types.CodeType(*args)
if hasattr(code_obj, "replace"):
# Python 3.8+
return code_obj.replace(
co_argcount=co_argcount + new_locals_len,
co_nlocals=code_obj.co_nlocals + new_locals_len,
co_code=code,
co_names=names,
co_varnames=varnames)
else:
return types.CodeType(*args)
def instructions(code_obj):

83
lib/jsc.py
View File

@@ -1,83 +0,0 @@
MAPPING = {
'a': '(false+"")[1]',
'b': '([]["entries"]()+"")[2]',
'c': '([]["fill"]+"")[3]',
'd': '(undefined+"")[2]',
'e': '(true+"")[3]',
'f': '(false+"")[0]',
'g': '(false+[0]+String)[20]',
'h': '(+(101))["to"+String["name"]](21)[1]',
'i': '([false]+undefined)[10]',
'j': '([]["entries"]()+"")[3]',
'k': '(+(20))["to"+String["name"]](21)',
'l': '(false+"")[2]',
'm': '(Number+"")[11]',
'n': '(undefined+"")[1]',
'o': '(true+[]["fill"])[10]',
'p': '(+(211))["to"+String["name"]](31)[1]',
'q': '(+(212))["to"+String["name"]](31)[1]',
'r': '(true+"")[1]',
's': '(false+"")[3]',
't': '(true+"")[0]',
'u': '(undefined+"")[0]',
'v': '(+(31))["to"+String["name"]](32)',
'w': '(+(32))["to"+String["name"]](33)',
'x': '(+(101))["to"+String["name"]](34)[1]',
'y': '(NaN+[Infinity])[10]',
'z': '(+(35))["to"+String["name"]](36)',
'A': '(+[]+Array)[10]',
'B': '(+[]+Boolean)[10]',
'C': 'Function("return escape")()(("")["italics"]())[2]',
'D': 'Function("return escape")()([]["fill"])["slice"]("-1")',
'E': '(RegExp+"")[12]',
'F': '(+[]+Function)[10]',
'G': '(false+Function("return Date")()())[30]',
'I': '(Infinity+"")[0]',
'M': '(true+Function("return Date")()())[30]',
'N': '(NaN+"")[0]',
'O': '(NaN+Function("return{}")())[11]',
'R': '(+[]+RegExp)[10]',
'S': '(+[]+String)[10]',
'T': '(NaN+Function("return Date")()())[30]',
'U': '(NaN+Function("return{}")()["to"+String["name"]]["call"]())[11]',
' ': '(NaN+[]["fill"])[11]',
'"': '("")["fontcolor"]()[12]',
'%': 'Function("return escape")()([]["fill"])[21]',
'&': '("")["link"](0+")[10]',
'(': '(undefined+[]["fill"])[22]',
')': '([0]+false+[]["fill"])[20]',
'+': '(+(+!+[]+(!+[]+[])[!+[]+!+[]+!+[]]+[+!+[]]+[+[]]+[+[]])+[])[2]',
',': '([]["slice"]["call"](false+"")+"")[1]',
'-': '(+(.+[0000000001])+"")[2]',
'.': '(+(+!+[]+[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+[!+[]+!+[]]+[+[]])+[])[+!+[]]',
'/': '(false+[0])["italics"]()[10]',
':': '(RegExp()+"")[3]',
';': '("")["link"](")[14]',
'<': '("")["italics"]()[0]',
'=': '("")["fontcolor"]()[11]',
'>': '("")["italics"]()[2]',
'?': '(RegExp()+"")[2]',
'[': '([]["entries"]()+"")[0]',
']': '([]["entries"]()+"")[22]',
'{': '(true+[]["fill"])[20]',
'}': '([]["fill"]+"")["slice"]("-1")'
}
SIMPLE = {
'false': '![]',
'true': '!![]',
'undefined': '[][[]]',
'NaN': '+[![]]',
'Infinity': '+(+!+[]+(!+[]+[])[!+[]+!+[]+!+[]]+[+!+[]]+[+[]]+[+[]]+[+[]])' # +"1e1000"
}
def jsunc(jscString):
for key in sorted(MAPPING, key=lambda k: len(MAPPING[k]), reverse=True):
if MAPPING.get(key) in jscString:
jscString = jscString.replace(MAPPING.get(key), '"{}"'.format(key))
for key in sorted(SIMPLE, key=lambda k: len(SIMPLE[k]), reverse=True):
if SIMPLE.get(key) in jscString:
jscString = jscString.replace(SIMPLE.get(key), '{}'.format(key))
return jscString

550
lib/jscrypto.py
View File

@@ -1,550 +0,0 @@
# -*- coding: utf-8 -*-
import StringIO
import binascii
import hashlib
from array import array
def evpKDF(passwd, salt, key_size=8, iv_size=4, iterations=1, hash_algorithm="md5"):
target_key_size = key_size + iv_size
derived_bytes = ""
number_of_derived_words = 0
block = None
hasher = hashlib.new(hash_algorithm)
while number_of_derived_words < target_key_size:
if block is not None:
hasher.update(block)
hasher.update(passwd)
hasher.update(salt)
block = hasher.digest()
hasher = hashlib.new(hash_algorithm)
for i in range(1, iterations):
hasher.update(block)
block = hasher.digest()
hasher = hashlib.new(hash_algorithm)
derived_bytes += block[0: min(len(block), (target_key_size - number_of_derived_words) * 4)]
number_of_derived_words += len(block) / 4
return {
"key": derived_bytes[0: key_size * 4],
"iv": derived_bytes[key_size * 4:]
}
class PKCS7Encoder(object):
'''
RFC 2315: PKCS#7 page 21
Some content-encryption algorithms assume the
input length is a multiple of k octets, where k > 1, and
let the application define a method for handling inputs
whose lengths are not a multiple of k octets. For such
algorithms, the method shall be to pad the input at the
trailing end with k - (l mod k) octets all having value k -
(l mod k), where l is the length of the input. In other
words, the input is padded at the trailing end with one of
the following strings:
01 -- if l mod k = k-1
02 02 -- if l mod k = k-2
.
.
.
k k ... k k -- if l mod k = 0
The padding can be removed unambiguously since all input is
padded and no padding string is a suffix of another. This
padding method is well-defined if and only if k < 256;
methods for larger k are an open issue for further study.
'''
def __init__(self, k=16):
self.k = k
## @param text The padded text for which the padding is to be removed.
# @exception ValueError Raised when the input padding is missing or corrupt.
def decode(self, text):
'''
Remove the PKCS#7 padding from a text string
'''
nl = len(text)
val = int(binascii.hexlify(text[-1]), 16)
if val > self.k:
raise ValueError('Input is not padded or padding is corrupt')
l = nl - val
return text[:l]
## @param text The text to encode.
def encode(self, text):
'''
Pad an input string according to PKCS#7
'''
l = len(text)
output = StringIO.StringIO()
val = self.k - (l % self.k)
for _ in xrange(val):
output.write('%02x' % val)
return text + binascii.unhexlify(output.getvalue())
# Pyaes file
# Globals mandated by PEP 272:
# http://www.python.org/dev/peps/pep-0272/
MODE_ECB = 1
MODE_CBC = 2
# MODE_CTR = 6
block_size = 16
key_size = None
def new(key, mode, IV=None):
if mode == MODE_ECB:
return ECBMode(AES(key))
elif mode == MODE_CBC:
if IV is None:
raise ValueError, "CBC mode needs an IV value!"
return CBCMode(AES(key), IV)
else:
raise NotImplementedError
#### AES cipher implementation
class AES(object):
block_size = 16
def __init__(self, key):
self.setkey(key)
def setkey(self, key):
"""Sets the key and performs key expansion."""
self.key = key
self.key_size = len(key)
if self.key_size == 16:
self.rounds = 10
elif self.key_size == 24:
self.rounds = 12
elif self.key_size == 32:
self.rounds = 14
else:
raise ValueError, "Key length must be 16, 24 or 32 bytes"
self.expand_key()
def expand_key(self):
"""Performs AES key expansion on self.key and stores in self.exkey"""
# The key schedule specifies how parts of the key are fed into the
# cipher's round functions. "Key expansion" means performing this
# schedule in advance. Almost all implementations do this.
#
# Here's a description of AES key schedule:
# http://en.wikipedia.org/wiki/Rijndael_key_schedule
# The expanded key starts with the actual key itself
exkey = array('B', self.key)
# extra key expansion steps
if self.key_size == 16:
extra_cnt = 0
elif self.key_size == 24:
extra_cnt = 2
else:
extra_cnt = 3
# 4-byte temporary variable for key expansion
word = exkey[-4:]
# Each expansion cycle uses 'i' once for Rcon table lookup
for i in xrange(1, 11):
#### key schedule core:
# left-rotate by 1 byte
word = word[1:4] + word[0:1]
# apply S-box to all bytes
for j in xrange(4):
word[j] = aes_sbox[word[j]]
# apply the Rcon table to the leftmost byte
word[0] = word[0] ^ aes_Rcon[i]
#### end key schedule core
for z in xrange(4):
for j in xrange(4):
# mix in bytes from the last subkey
word[j] ^= exkey[-self.key_size + j]
exkey.extend(word)
# Last key expansion cycle always finishes here
if len(exkey) >= (self.rounds + 1) * self.block_size:
break
# Special substitution step for 256-bit key
if self.key_size == 32:
for j in xrange(4):
# mix in bytes from the last subkey XORed with S-box of
# current word bytes
word[j] = aes_sbox[word[j]] ^ exkey[-self.key_size + j]
exkey.extend(word)
# Twice for 192-bit key, thrice for 256-bit key
for z in xrange(extra_cnt):
for j in xrange(4):
# mix in bytes from the last subkey
word[j] ^= exkey[-self.key_size + j]
exkey.extend(word)
self.exkey = exkey
def add_round_key(self, block, round):
"""AddRoundKey step in AES. This is where the key is mixed into plaintext"""
offset = round * 16
exkey = self.exkey
for i in xrange(16):
block[i] ^= exkey[offset + i]
# print 'AddRoundKey:', block
def sub_bytes(self, block, sbox):
"""SubBytes step, apply S-box to all bytes
Depending on whether encrypting or decrypting, a different sbox array
is passed in.
"""
for i in xrange(16):
block[i] = sbox[block[i]]
# print 'SubBytes :', block
def shift_rows(self, b):
"""ShiftRows step. Shifts 2nd row to left by 1, 3rd row by 2, 4th row by 3
Since we're performing this on a transposed matrix, cells are numbered
from top to bottom::
0 4 8 12 -> 0 4 8 12 -- 1st row doesn't change
1 5 9 13 -> 5 9 13 1 -- row shifted to left by 1 (wraps around)
2 6 10 14 -> 10 14 2 6 -- shifted by 2
3 7 11 15 -> 15 3 7 11 -- shifted by 3
"""
b[1], b[5], b[9], b[13] = b[5], b[9], b[13], b[1]
b[2], b[6], b[10], b[14] = b[10], b[14], b[2], b[6]
b[3], b[7], b[11], b[15] = b[15], b[3], b[7], b[11]
# print 'ShiftRows :', b
def shift_rows_inv(self, b):
"""Similar to shift_rows above, but performed in inverse for decryption."""
b[5], b[9], b[13], b[1] = b[1], b[5], b[9], b[13]
b[10], b[14], b[2], b[6] = b[2], b[6], b[10], b[14]
b[15], b[3], b[7], b[11] = b[3], b[7], b[11], b[15]
# print 'ShiftRows :', b
def mix_columns(self, block):
"""MixColumns step. Mixes the values in each column"""
# Cache global multiplication tables (see below)
mul_by_2 = gf_mul_by_2
mul_by_3 = gf_mul_by_3
# Since we're dealing with a transposed matrix, columns are already
# sequential
for i in xrange(4):
col = i * 4
# v0, v1, v2, v3 = block[col : col+4]
v0, v1, v2, v3 = (block[col], block[col + 1], block[col + 2],
block[col + 3])
block[col] = mul_by_2[v0] ^ v3 ^ v2 ^ mul_by_3[v1]
block[col + 1] = mul_by_2[v1] ^ v0 ^ v3 ^ mul_by_3[v2]
block[col + 2] = mul_by_2[v2] ^ v1 ^ v0 ^ mul_by_3[v3]
block[col + 3] = mul_by_2[v3] ^ v2 ^ v1 ^ mul_by_3[v0]
# print 'MixColumns :', block
def mix_columns_inv(self, block):
"""Similar to mix_columns above, but performed in inverse for decryption."""
# Cache global multiplication tables (see below)
mul_9 = gf_mul_by_9
mul_11 = gf_mul_by_11
mul_13 = gf_mul_by_13
mul_14 = gf_mul_by_14
# Since we're dealing with a transposed matrix, columns are already
# sequential
for i in xrange(4):
col = i * 4
v0, v1, v2, v3 = (block[col], block[col + 1], block[col + 2],
block[col + 3])
# v0, v1, v2, v3 = block[col:col+4]
block[col] = mul_14[v0] ^ mul_9[v3] ^ mul_13[v2] ^ mul_11[v1]
block[col + 1] = mul_14[v1] ^ mul_9[v0] ^ mul_13[v3] ^ mul_11[v2]
block[col + 2] = mul_14[v2] ^ mul_9[v1] ^ mul_13[v0] ^ mul_11[v3]
block[col + 3] = mul_14[v3] ^ mul_9[v2] ^ mul_13[v1] ^ mul_11[v0]
# print 'MixColumns :', block
def encrypt_block(self, block):
"""Encrypts a single block. This is the main AES function"""
# For efficiency reasons, the state between steps is transmitted via a
# mutable array, not returned.
self.add_round_key(block, 0)
for round in xrange(1, self.rounds):
self.sub_bytes(block, aes_sbox)
self.shift_rows(block)
self.mix_columns(block)
self.add_round_key(block, round)
self.sub_bytes(block, aes_sbox)
self.shift_rows(block)
# no mix_columns step in the last round
self.add_round_key(block, self.rounds)
def decrypt_block(self, block):
"""Decrypts a single block. This is the main AES decryption function"""
# For efficiency reasons, the state between steps is transmitted via a
# mutable array, not returned.
self.add_round_key(block, self.rounds)
# count rounds down from 15 ... 1
for round in xrange(self.rounds - 1, 0, -1):
self.shift_rows_inv(block)
self.sub_bytes(block, aes_inv_sbox)
self.add_round_key(block, round)
self.mix_columns_inv(block)
self.shift_rows_inv(block)
self.sub_bytes(block, aes_inv_sbox)
self.add_round_key(block, 0)
# no mix_columns step in the last round
#### ECB mode implementation
class ECBMode(object):
"""Electronic CodeBook (ECB) mode encryption.
Basically this mode applies the cipher function to each block individually;
no feedback is done. NB! This is insecure for almost all purposes
"""
def __init__(self, cipher):
self.cipher = cipher
self.block_size = cipher.block_size
def ecb(self, data, block_func):
"""Perform ECB mode with the given function"""
if len(data) % self.block_size != 0:
raise ValueError, "Plaintext length must be multiple of 16"
block_size = self.block_size
data = array('B', data)
for offset in xrange(0, len(data), block_size):
block = data[offset: offset + block_size]
block_func(block)
data[offset: offset + block_size] = block
return data.tostring()
def encrypt(self, data):
"""Encrypt data in ECB mode"""
return self.ecb(data, self.cipher.encrypt_block)
def decrypt(self, data):
"""Decrypt data in ECB mode"""
return self.ecb(data, self.cipher.decrypt_block)
#### CBC mode
class CBCMode(object):
"""Cipher Block Chaining (CBC) mode encryption. This mode avoids content leaks.
In CBC encryption, each plaintext block is XORed with the ciphertext block
preceding it; decryption is simply the inverse.
"""
# A better explanation of CBC can be found here:
# http://en.wikipedia.org/wiki/Block_cipher_modes_of_operation#Cipher-block_chaining_.28CBC.29
def __init__(self, cipher, IV):
self.cipher = cipher
self.block_size = cipher.block_size
self.IV = array('B', IV)
def encrypt(self, data):
"""Encrypt data in CBC mode"""
block_size = self.block_size
if len(data) % block_size != 0:
raise ValueError, "Plaintext length must be multiple of 16"
data = array('B', data)
IV = self.IV
for offset in xrange(0, len(data), block_size):
block = data[offset: offset + block_size]
# Perform CBC chaining
for i in xrange(block_size):
block[i] ^= IV[i]
self.cipher.encrypt_block(block)
data[offset: offset + block_size] = block
IV = block
self.IV = IV
return data.tostring()
def decrypt(self, data):
"""Decrypt data in CBC mode"""
block_size = self.block_size
if len(data) % block_size != 0:
raise ValueError, "Ciphertext length must be multiple of 16"
data = array('B', data)
IV = self.IV
for offset in xrange(0, len(data), block_size):
ctext = data[offset: offset + block_size]
block = ctext[:]
self.cipher.decrypt_block(block)
# Perform CBC chaining
# for i in xrange(block_size):
# data[offset + i] ^= IV[i]
for i in xrange(block_size):
block[i] ^= IV[i]
data[offset: offset + block_size] = block
IV = ctext
# data[offset : offset+block_size] = block
self.IV = IV
return data.tostring()
####
def galois_multiply(a, b):
"""Galois Field multiplicaiton for AES"""
p = 0
while b:
if b & 1:
p ^= a
a <<= 1
if a & 0x100:
a ^= 0x1b
b >>= 1
return p & 0xff
# Precompute the multiplication tables for encryption
gf_mul_by_2 = array('B', [galois_multiply(x, 2) for x in range(256)])
gf_mul_by_3 = array('B', [galois_multiply(x, 3) for x in range(256)])
# ... for decryption
gf_mul_by_9 = array('B', [galois_multiply(x, 9) for x in range(256)])
gf_mul_by_11 = array('B', [galois_multiply(x, 11) for x in range(256)])
gf_mul_by_13 = array('B', [galois_multiply(x, 13) for x in range(256)])
gf_mul_by_14 = array('B', [galois_multiply(x, 14) for x in range(256)])
####
# The S-box is a 256-element array, that maps a single byte value to another
# byte value. Since it's designed to be reversible, each value occurs only once
# in the S-box
#
# More information: http://en.wikipedia.org/wiki/Rijndael_S-box
aes_sbox = array('B',
'637c777bf26b6fc53001672bfed7ab76'
'ca82c97dfa5947f0add4a2af9ca472c0'
'b7fd9326363ff7cc34a5e5f171d83115'
'04c723c31896059a071280e2eb27b275'
'09832c1a1b6e5aa0523bd6b329e32f84'
'53d100ed20fcb15b6acbbe394a4c58cf'
'd0efaafb434d338545f9027f503c9fa8'
'51a3408f929d38f5bcb6da2110fff3d2'
'cd0c13ec5f974417c4a77e3d645d1973'
'60814fdc222a908846eeb814de5e0bdb'
'e0323a0a4906245cc2d3ac629195e479'
'e7c8376d8dd54ea96c56f4ea657aae08'
'ba78252e1ca6b4c6e8dd741f4bbd8b8a'
'703eb5664803f60e613557b986c11d9e'
'e1f8981169d98e949b1e87e9ce5528df'
'8ca1890dbfe6426841992d0fb054bb16'.decode('hex')
)
# This is the inverse of the above. In other words:
# aes_inv_sbox[aes_sbox[val]] == val
aes_inv_sbox = array('B',
'52096ad53036a538bf40a39e81f3d7fb'
'7ce339829b2fff87348e4344c4dee9cb'
'547b9432a6c2233dee4c950b42fac34e'
'082ea16628d924b2765ba2496d8bd125'
'72f8f66486689816d4a45ccc5d65b692'
'6c704850fdedb9da5e154657a78d9d84'
'90d8ab008cbcd30af7e45805b8b34506'
'd02c1e8fca3f0f02c1afbd0301138a6b'
'3a9111414f67dcea97f2cfcef0b4e673'
'96ac7422e7ad3585e2f937e81c75df6e'
'47f11a711d29c5896fb7620eaa18be1b'
'fc563e4bc6d279209adbc0fe78cd5af4'
'1fdda8338807c731b11210592780ec5f'
'60517fa919b54a0d2de57a9f93c99cef'
'a0e03b4dae2af5b0c8ebbb3c83539961'
'172b047eba77d626e169146355210c7d'.decode('hex')
)
# The Rcon table is used in AES's key schedule (key expansion)
# It's a pre-computed table of exponentation of 2 in AES's finite field
#
# More information: http://en.wikipedia.org/wiki/Rijndael_key_schedule
aes_Rcon = array('B',
'8d01020408102040801b366cd8ab4d9a'
'2f5ebc63c697356ad4b37dfaefc59139'
'72e4d3bd61c29f254a943366cc831d3a'
'74e8cb8d01020408102040801b366cd8'
'ab4d9a2f5ebc63c697356ad4b37dfaef'
'c5913972e4d3bd61c29f254a943366cc'
'831d3a74e8cb8d01020408102040801b'
'366cd8ab4d9a2f5ebc63c697356ad4b3'
'7dfaefc5913972e4d3bd61c29f254a94'
'3366cc831d3a74e8cb8d010204081020'
'40801b366cd8ab4d9a2f5ebc63c69735'
'6ad4b37dfaefc5913972e4d3bd61c29f'
'254a943366cc831d3a74e8cb8d010204'
'08102040801b366cd8ab4d9a2f5ebc63'
'c697356ad4b37dfaefc5913972e4d3bd'
'61c29f254a943366cc831d3a74e8cb'.decode('hex')
)

249
lib/jsinterpreter.py
View File

@@ -1,249 +0,0 @@
# -*- coding: utf-8 -*-
import json
import operator
import re
_OPERATORS = [
('|', operator.or_),
('^', operator.xor),
('&', operator.and_),
('>>', operator.rshift),
('<<', operator.lshift),
('-', operator.sub),
('+', operator.add),
('%', operator.mod),
('/', operator.truediv),
('*', operator.mul),
]
_ASSIGN_OPERATORS = []
for op, opfunc in _OPERATORS:
_ASSIGN_OPERATORS.append([op + '=', opfunc])
_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
class JSInterpreter(object):
def __init__(self, code, objects=None):
if objects is None:
objects = {}
self.code = code
self._functions = {}
self._objects = objects
def interpret_statement(self, stmt, local_vars, allow_recursion=100):
should_abort = False
stmt = stmt.lstrip()
stmt_m = re.match(r'var\s', stmt)
if stmt_m:
expr = stmt[len(stmt_m.group(0)):]
else:
return_m = re.match(r'return(?:\s+|$)', stmt)
if return_m:
expr = stmt[len(return_m.group(0)):]
should_abort = True
else:
# Try interpreting it as an expression
expr = stmt
v = self.interpret_expression(expr, local_vars, allow_recursion)
return v, should_abort
def interpret_expression(self, expr, local_vars, allow_recursion):
expr = expr.strip()
if expr == '': # Empty expression
return None
if expr.startswith('('):
parens_count = 0
for m in re.finditer(r'[()]', expr):
if m.group(0) == '(':
parens_count += 1
else:
parens_count -= 1
if parens_count == 0:
sub_expr = expr[1:m.start()]
sub_result = self.interpret_expression(
sub_expr, local_vars, allow_recursion)
remaining_expr = expr[m.end():].strip()
if not remaining_expr:
return sub_result
else:
expr = json.dumps(sub_result) + remaining_expr
break
for op, opfunc in _ASSIGN_OPERATORS:
m = re.match(r'''(?x)
(?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
\s*%s
(?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
if not m:
continue
right_val = self.interpret_expression(
m.group('expr'), local_vars, allow_recursion - 1)
if m.groupdict().get('index'):
lvar = local_vars[m.group('out')]
idx = self.interpret_expression(
m.group('index'), local_vars, allow_recursion)
assert isinstance(idx, int)
cur = lvar[idx]
val = opfunc(cur, right_val)
lvar[idx] = val
return val
else:
cur = local_vars.get(m.group('out'))
val = opfunc(cur, right_val)
local_vars[m.group('out')] = val
return val
if expr.isdigit():
return int(expr)
var_m = re.match(
r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
expr)
if var_m:
return local_vars[var_m.group('name')]
try:
return json.loads(expr)
except ValueError:
pass
m = re.match(
r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
expr)
if m:
variable = m.group('var')
member = m.group('member')
arg_str = m.group('args')
if variable in local_vars:
obj = local_vars[variable]
else:
if variable not in self._objects:
self._objects[variable] = self.extract_object(variable)
obj = self._objects[variable]
if arg_str is None:
# Member access
if member == 'length':
return len(obj)
return obj[member]
assert expr.endswith(')')
# Function call
if arg_str == '':
argvals = tuple()
else:
argvals = []
for v in arg_str.split(','):
argvals.extend([self.interpret_expression(v, local_vars, allow_recursion)])
if member == 'split':
assert argvals == ('',)
return list(obj)
if member == 'join':
assert len(argvals) == 1
return argvals[0].join(obj)
if member == 'reverse':
assert len(argvals) == 0
obj.reverse()
return obj
if member == 'slice':
assert len(argvals) == 1
return obj[argvals[0]:]
if member == 'splice':
assert isinstance(obj, list)
index, howMany = argvals
res = []
for i in range(index, min(index + howMany, len(obj))):
res.append(obj.pop(index))
return res
return obj[member](argvals)
m = re.match(
r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
if m:
val = local_vars[m.group('in')]
idx = self.interpret_expression(
m.group('idx'), local_vars, allow_recursion - 1)
return val[idx]
for op, opfunc in _OPERATORS:
m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
if not m:
continue
x, abort = self.interpret_statement(
m.group('x'), local_vars, allow_recursion - 1)
y, abort = self.interpret_statement(
m.group('y'), local_vars, allow_recursion - 1)
return opfunc(x, y)
m = re.match(
r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]+)\)$' % _NAME_RE, expr)
if m:
fname = m.group('func')
argvals = []
for v in m.group('args').split(','):
if v.isdigit():
argvals.append([int(v)])
else:
argvals.append([local_vars[v]])
if fname not in self._functions:
self._functions[fname] = self.extract_function(fname)
return self._functions[fname](argvals)
def extract_object(self, objname):
obj = {}
obj_m = re.search(
(r'(?:var\s+)?%s\s*=\s*\{' % re.escape(objname)) +
r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
r'\}\s*;',
self.code)
fields = obj_m.group('fields')
# Currently, it only supports function definitions
fields_m = re.finditer(
r'(?P<key>[a-zA-Z$0-9]+)\s*:\s*function'
r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
fields)
for f in fields_m:
argnames = f.group('args').split(',')
obj[f.group('key')] = self.build_function(argnames, f.group('code'))
return obj
def extract_function(self, funcname):
func_m = re.search(
r'''(?x)
(?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
\((?P<args>[^)]*)\)\s*
\{(?P<code>[^}]+)\}''' % (
re.escape(funcname), re.escape(funcname), re.escape(funcname)),
self.code)
argnames = func_m.group('args').split(',')
return self.build_function(argnames, func_m.group('code'))
def call_function(self, funcname, *args):
f = self.extract_function(funcname)
return f(args)
def build_function(self, argnames, code):
def resf(args):
local_vars = dict(zip(argnames, args))
for stmt in code.split(';'):
res, abort = self.interpret_statement(stmt, local_vars)
if abort:
break
return res
return resf

15
lib/pafy/__init__.py
View File

@@ -1,15 +0,0 @@
from .pafy import get_playlist
from .pafy import new
from .pafy import set_api_key
from .pafy import dump_cache
from .pafy import load_cache
from .pafy import get_categoryname
from .pafy import __version__
from .pafy import __author__
from .pafy import __license__
import sys
if "test" not in sys.argv[0]:
del pafy
del sys

1618
lib/pafy/pafy.py
View File

File diff suppressed because it is too large Load Diff

5
lib/pyjsparser/__init__.py
View File

@@ -1,4 +1,5 @@
__all__ = ['PyJsParser', 'parse', 'JsSyntaxError']
__all__ = ['PyJsParser', 'parse', 'JsSyntaxError', 'pyjsparserdata']
__author__ = 'Piotr Dabkowski'
__version__ = '2.2.0'
from .parser import PyJsParser, parse, JsSyntaxError
from .parser import PyJsParser, parse, JsSyntaxError
from . import pyjsparserdata

1468
lib/pyjsparser/parser.py
View File

File diff suppressed because it is too large Load Diff

492
lib/pyjsparser/pyjsparserdata.py
View File

@@ -23,7 +23,7 @@ import sys
import unicodedata
from collections import defaultdict
PY3 = sys.version_info >= (3,0)
PY3 = sys.version_info >= (3, 0)
if PY3:
unichr = chr
@@ -31,179 +31,254 @@ if PY3:
unicode = str
token = {
'BooleanLiteral': 1,
'EOF': 2,
'Identifier': 3,
'Keyword': 4,
'NullLiteral': 5,
'NumericLiteral': 6,
'Punctuator': 7,
'StringLiteral': 8,
'RegularExpression': 9,
'Template': 10
}
'BooleanLiteral': 1,
'EOF': 2,
'Identifier': 3,
'Keyword': 4,
'NullLiteral': 5,
'NumericLiteral': 6,
'Punctuator': 7,
'StringLiteral': 8,
'RegularExpression': 9,
'Template': 10
}
TokenName = dict((v, k) for k, v in token.items())
TokenName = dict((v,k) for k,v in token.items())
FnExprTokens = [
'(',
'{',
'[',
'in',
'typeof',
'instanceof',
'new',
'return',
'case',
'delete',
'throw',
'void',
# assignment operators
'=',
'+=',
'-=',
'*=',
'/=',
'%=',
'<<=',
'>>=',
'>>>=',
'&=',
'|=',
'^=',
',',
# binary/unary operators
'+',
'-',
'*',
'/',
'%',
'++',
'--',
'<<',
'>>',
'>>>',
'&',
'|',
'^',
'!',
'~',
'&&',
'||',
'?',
':',
'===',
'==',
'>=',
'<=',
'<',
'>',
'!=',
'!=='
]
FnExprTokens = ['(', '{', '[', 'in', 'typeof', 'instanceof', 'new',
'return', 'case', 'delete', 'throw', 'void',
# assignment operators
'=', '+=', '-=', '*=', '/=', '%=', '<<=', '>>=', '>>>=',
'&=', '|=', '^=', ',',
# binary/unary operators
'+', '-', '*', '/', '%', '++', '--', '<<', '>>', '>>>', '&',
'|', '^', '!', '~', '&&', '||', '?', ':', '===', '==', '>=',
'<=', '<', '>', '!=', '!==']
syntax= set(('AssignmentExpression',
'AssignmentPattern',
'ArrayExpression',
'ArrayPattern',
'ArrowFunctionExpression',
'BlockStatement',
'BinaryExpression',
'BreakStatement',
'CallExpression',
'CatchClause',
'ClassBody',
'ClassDeclaration',
'ClassExpression',
'ConditionalExpression',
'ContinueStatement',
'DoWhileStatement',
'DebuggerStatement',
'EmptyStatement',
'ExportAllDeclaration',
'ExportDefaultDeclaration',
'ExportNamedDeclaration',
'ExportSpecifier',
'ExpressionStatement',
'ForStatement',
'ForInStatement',
'FunctionDeclaration',
'FunctionExpression',
'Identifier',
'IfStatement',
'ImportDeclaration',
'ImportDefaultSpecifier',
'ImportNamespaceSpecifier',
'ImportSpecifier',
'Literal',
'LabeledStatement',
'LogicalExpression',
'MemberExpression',
'MethodDefinition',
'NewExpression',
'ObjectExpression',
'ObjectPattern',
'Program',
'Property',
'RestElement',
'ReturnStatement',
'SequenceExpression',
'SpreadElement',
'Super',
'SwitchCase',
'SwitchStatement',
'TaggedTemplateExpression',
'TemplateElement',
'TemplateLiteral',
'ThisExpression',
'ThrowStatement',
'TryStatement',
'UnaryExpression',
'UpdateExpression',
'VariableDeclaration',
'VariableDeclarator',
'WhileStatement',
'WithStatement'))
syntax = set(
('AssignmentExpression', 'AssignmentPattern', 'ArrayExpression',
'ArrayPattern', 'ArrowFunctionExpression', 'BlockStatement',
'BinaryExpression', 'BreakStatement', 'CallExpression', 'CatchClause',
'ClassBody', 'ClassDeclaration', 'ClassExpression',
'ConditionalExpression', 'ContinueStatement', 'DoWhileStatement',
'DebuggerStatement', 'EmptyStatement', 'ExportAllDeclaration',
'ExportDefaultDeclaration', 'ExportNamedDeclaration', 'ExportSpecifier',
'ExpressionStatement', 'ForStatement', 'ForInStatement',
'FunctionDeclaration', 'FunctionExpression', 'Identifier', 'IfStatement',
'ImportDeclaration', 'ImportDefaultSpecifier', 'ImportNamespaceSpecifier',
'ImportSpecifier', 'Literal', 'LabeledStatement', 'LogicalExpression',
'MemberExpression', 'MethodDefinition', 'NewExpression',
'ObjectExpression', 'ObjectPattern', 'Program', 'Property', 'RestElement',
'ReturnStatement', 'SequenceExpression', 'SpreadElement', 'Super',
'SwitchCase', 'SwitchStatement', 'TaggedTemplateExpression',
'TemplateElement', 'TemplateLiteral', 'ThisExpression', 'ThrowStatement',
'TryStatement', 'UnaryExpression', 'UpdateExpression',
'VariableDeclaration', 'VariableDeclarator', 'WhileStatement',
'WithStatement'))
supported_syntax = set(
('AssignmentExpression', 'ArrayExpression', 'BlockStatement',
'BinaryExpression', 'BreakStatement', 'CallExpression', 'CatchClause',
'ConditionalExpression', 'ContinueStatement', 'DoWhileStatement',
'DebuggerStatement', 'EmptyStatement', 'ExpressionStatement',
'ForStatement', 'ForInStatement', 'FunctionDeclaration',
'FunctionExpression', 'Identifier', 'IfStatement', 'Literal',
'LabeledStatement', 'LogicalExpression', 'MemberExpression',
'MethodDefinition', 'NewExpression', 'ObjectExpression', 'Program',
'Property', 'ReturnStatement', 'SequenceExpression', 'SwitchCase',
'SwitchStatement', 'ThisExpression', 'ThrowStatement', 'TryStatement',
'UnaryExpression', 'UpdateExpression', 'VariableDeclaration',
'VariableDeclarator', 'WhileStatement', 'WithStatement'))
# Error messages should be identical to V8.
messages = {
'UnexpectedToken': 'Unexpected token %s',
'UnexpectedNumber': 'Unexpected number',
'UnexpectedString': 'Unexpected string',
'UnexpectedIdentifier': 'Unexpected identifier',
'UnexpectedReserved': 'Unexpected reserved word',
'UnexpectedTemplate': 'Unexpected quasi %s',
'UnexpectedEOS': 'Unexpected end of input',
'NewlineAfterThrow': 'Illegal newline after throw',
'InvalidRegExp': 'Invalid regular expression',
'UnterminatedRegExp': 'Invalid regular expression: missing /',
'InvalidLHSInAssignment': 'Invalid left-hand side in assignment',
'InvalidLHSInForIn': 'Invalid left-hand side in for-in',
'MultipleDefaultsInSwitch': 'More than one default clause in switch statement',
'NoCatchOrFinally': 'Missing catch or finally after try',
'UnknownLabel': 'Undefined label \'%s\'',
'Redeclaration': '%s \'%s\' has already been declared',
'IllegalContinue': 'Illegal continue statement',
'IllegalBreak': 'Illegal break statement',
'IllegalReturn': 'Illegal return statement',
'StrictModeWith': 'Strict mode code may not include a with statement',
'StrictCatchVariable': 'Catch variable may not be eval or arguments in strict mode',
'StrictVarName': 'Variable name may not be eval or arguments in strict mode',
'StrictParamName': 'Parameter name eval or arguments is not allowed in strict mode',
'StrictParamDupe': 'Strict mode function may not have duplicate parameter names',
'StrictFunctionName': 'Function name may not be eval or arguments in strict mode',
'StrictOctalLiteral': 'Octal literals are not allowed in strict mode.',
'StrictDelete': 'Delete of an unqualified identifier in strict mode.',
'StrictLHSAssignment': 'Assignment to eval or arguments is not allowed in strict mode',
'StrictLHSPostfix': 'Postfix increment/decrement may not have eval or arguments operand in strict mode',
'StrictLHSPrefix': 'Prefix increment/decrement may not have eval or arguments operand in strict mode',
'StrictReservedWord': 'Use of future reserved word in strict mode',
'TemplateOctalLiteral': 'Octal literals are not allowed in template strings.',
'ParameterAfterRestParameter': 'Rest parameter must be last formal parameter',
'DefaultRestParameter': 'Unexpected token =',
'ObjectPatternAsRestParameter': 'Unexpected token {',
'DuplicateProtoProperty': 'Duplicate __proto__ fields are not allowed in object literals',
'ConstructorSpecialMethod': 'Class constructor may not be an accessor',
'DuplicateConstructor': 'A class may only have one constructor',
'StaticPrototype': 'Classes may not have static property named prototype',
'MissingFromClause': 'Unexpected token',
'NoAsAfterImportNamespace': 'Unexpected token',
'InvalidModuleSpecifier': 'Unexpected token',
'IllegalImportDeclaration': 'Unexpected token',
'IllegalExportDeclaration': 'Unexpected token'}
'UnexpectedToken':
'Unexpected token %s',
'UnexpectedNumber':
'Unexpected number',
'UnexpectedString':
'Unexpected string',
'UnexpectedIdentifier':
'Unexpected identifier',
'UnexpectedReserved':
'Unexpected reserved word',
'UnexpectedTemplate':
'Unexpected quasi %s',
'UnexpectedEOS':
'Unexpected end of input',
'NewlineAfterThrow':
'Illegal newline after throw',
'InvalidRegExp':
'Invalid regular expression',
'UnterminatedRegExp':
'Invalid regular expression: missing /',
'InvalidLHSInAssignment':
'Invalid left-hand side in assignment',
'InvalidLHSInForIn':
'Invalid left-hand side in for-in',
'MultipleDefaultsInSwitch':
'More than one default clause in switch statement',
'NoCatchOrFinally':
'Missing catch or finally after try',
'UnknownLabel':
'Undefined label \'%s\'',
'Redeclaration':
'%s \'%s\' has already been declared',
'IllegalContinue':
'Illegal continue statement',
'IllegalBreak':
'Illegal break statement',
'IllegalReturn':
'Illegal return statement',
'StrictModeWith':
'Strict mode code may not include a with statement',
'StrictCatchVariable':
'Catch variable may not be eval or arguments in strict mode',
'StrictVarName':
'Variable name may not be eval or arguments in strict mode',
'StrictParamName':
'Parameter name eval or arguments is not allowed in strict mode',
'StrictParamDupe':
'Strict mode function may not have duplicate parameter names',
'StrictFunctionName':
'Function name may not be eval or arguments in strict mode',
'StrictOctalLiteral':
'Octal literals are not allowed in strict mode.',
'StrictDelete':
'Delete of an unqualified identifier in strict mode.',
'StrictLHSAssignment':
'Assignment to eval or arguments is not allowed in strict mode',
'StrictLHSPostfix':
'Postfix increment/decrement may not have eval or arguments operand in strict mode',
'StrictLHSPrefix':
'Prefix increment/decrement may not have eval or arguments operand in strict mode',
'StrictReservedWord':
'Use of future reserved word in strict mode',
'TemplateOctalLiteral':
'Octal literals are not allowed in template strings.',
'ParameterAfterRestParameter':
'Rest parameter must be last formal parameter',
'DefaultRestParameter':
'Unexpected token =',
'ObjectPatternAsRestParameter':
'Unexpected token {',
'DuplicateProtoProperty':
'Duplicate __proto__ fields are not allowed in object literals',
'ConstructorSpecialMethod':
'Class constructor may not be an accessor',
'DuplicateConstructor':
'A class may only have one constructor',
'StaticPrototype':
'Classes may not have static property named prototype',
'MissingFromClause':
'Unexpected token',
'NoAsAfterImportNamespace':
'Unexpected token',
'InvalidModuleSpecifier':
'Unexpected token',
'IllegalImportDeclaration':
'Unexpected token',
'IllegalExportDeclaration':
'Unexpected token'
}
PRECEDENCE = {
'||': 1,
'&&': 2,
'|': 3,
'^': 4,
'&': 5,
'==': 6,
'!=': 6,
'===': 6,
'!==': 6,
'<': 7,
'>': 7,
'<=': 7,
'>=': 7,
'instanceof': 7,
'in': 7,
'<<': 8,
'>>': 8,
'>>>': 8,
'+': 9,
'-': 9,
'*': 11,
'/': 11,
'%': 11
}
class Token:
pass
class Syntax:
pass
class Messages:
pass
PRECEDENCE = {'||':1,
'&&':2,
'|':3,
'^':4,
'&':5,
'==':6,
'!=':6,
'===':6,
'!==':6,
'<':7,
'>':7,
'<=':7,
'>=':7,
'instanceof':7,
'in':7,
'<<':8,
'>>':8,
'>>>':8,
'+':9,
'-':9,
'*':11,
'/':11,
'%':11}
class Token: pass
class Syntax: pass
class Messages: pass
class PlaceHolders:
ArrowParameterPlaceHolder = 'ArrowParameterPlaceHolder'
for k,v in token.items():
for k, v in token.items():
setattr(Token, k, v)
for e in syntax:
setattr(Syntax, e, e)
for k,v in messages.items():
for k, v in messages.items():
setattr(Messages, k, v)
#http://stackoverflow.com/questions/14245893/efficiently-list-all-characters-in-a-given-unicode-category
@@ -220,84 +295,113 @@ CR = u'\u000D'
LS = u'\u2028'
PS = u'\u2029'
U_CATEGORIES = defaultdict(list)
for c in map(unichr, range(sys.maxunicode + 1)):
U_CATEGORIES[unicodedata.category(c)].append(c)
UNICODE_LETTER = set(U_CATEGORIES['Lu']+U_CATEGORIES['Ll']+
U_CATEGORIES['Lt']+U_CATEGORIES['Lm']+
U_CATEGORIES['Lo']+U_CATEGORIES['Nl'])
UNICODE_COMBINING_MARK = set(U_CATEGORIES['Mn']+U_CATEGORIES['Mc'])
UNICODE_DIGIT = set(U_CATEGORIES['Nd'])
UNICODE_CONNECTOR_PUNCTUATION = set(U_CATEGORIES['Pc'])
IDENTIFIER_START = UNICODE_LETTER.union(set(('$','_', '\\'))) # and some fucking unicode escape sequence
IDENTIFIER_PART = IDENTIFIER_START.union(UNICODE_COMBINING_MARK).union(UNICODE_DIGIT)\
.union(UNICODE_CONNECTOR_PUNCTUATION).union(set((ZWJ, ZWNJ)))
WHITE_SPACE = set((0x20, 0x09, 0x0B, 0x0C, 0xA0, 0x1680,
0x180E, 0x2000, 0x2001, 0x2002, 0x2003,
0x2004, 0x2005, 0x2006, 0x2007, 0x2008,
0x2009, 0x200A, 0x202F, 0x205F, 0x3000,
0xFEFF))
LETTER_CATEGORIES = set(['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl'])
COMBINING_MARK_CATEGORIES = set(['Mn', 'Mc'])
DIGIT_CATEGORIES = set(['Nd'])
CONNECTOR_PUNCTUATION_CATEGORIES = set(['Pc'])
IDENTIFIER_START_CATEGORIES = LETTER_CATEGORIES.copy() # and some fucking unicode escape sequence
IDENTIFIER_PART_CATEGORIES = IDENTIFIER_START_CATEGORIES.union(COMBINING_MARK_CATEGORIES).union(DIGIT_CATEGORIES)\
.union(CONNECTOR_PUNCTUATION_CATEGORIES)
EXTRA_IDENTIFIER_START_CHARS = set(('$','_', '\\'))
EXTRA_IDENTIFIER_PART_CHARS = EXTRA_IDENTIFIER_START_CHARS.union(set((ZWJ, ZWNJ)))
WHITE_SPACE = set((0x20, 0x09, 0x0B, 0x0C, 0xA0, 0x1680, 0x180E, 0x2000,
0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007,
0x2008, 0x2009, 0x200A, 0x202F, 0x205F, 0x3000, 0xFEFF))
LINE_TERMINATORS = set((0x0A, 0x0D, 0x2028, 0x2029))
def isIdentifierStart(ch):
return (ch if isinstance(ch, unicode) else unichr(ch)) in IDENTIFIER_START
uch = (ch if isinstance(ch, unicode) else unichr(ch))
return unicodedata.category(uch) in IDENTIFIER_START_CATEGORIES or uch in EXTRA_IDENTIFIER_START_CHARS
def isIdentifierPart(ch):
return (ch if isinstance(ch, unicode) else unichr(ch)) in IDENTIFIER_PART
uch = (ch if isinstance(ch, unicode) else unichr(ch))
return unicodedata.category(uch) in IDENTIFIER_PART_CATEGORIES or uch in EXTRA_IDENTIFIER_PART_CHARS
def isValidIdentifier(name):
if not name or isKeyword(name):
return False
check = isIdentifierStart
for e in name:
if not check(e):
return False
check = isIdentifierPart
return True
def isWhiteSpace(ch):
return (ord(ch) if isinstance(ch, unicode) else ch) in WHITE_SPACE
def isLineTerminator(ch):
return (ord(ch) if isinstance(ch, unicode) else ch) in LINE_TERMINATORS
return (ord(ch) if isinstance(ch, unicode) else ch) in LINE_TERMINATORS
OCTAL = set(('0', '1', '2', '3', '4', '5', '6', '7'))
DEC = set(('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'))
HEX = set('0123456789abcdefABCDEF')
HEX_CONV = dict(('0123456789abcdef'[n],n) for n in xrange(16))
for i,e in enumerate('ABCDEF', 10):
HEX_CONV = dict(('0123456789abcdef' [n], n) for n in xrange(16))
for i, e in enumerate('ABCDEF', 10):
HEX_CONV[e] = i
def isDecimalDigit(ch):
return (ch if isinstance(ch, unicode) else unichr(ch)) in DEC
def isHexDigit(ch):
return (ch if isinstance(ch, unicode) else unichr(ch)) in HEX
return (ch if isinstance(ch, unicode) else unichr(ch)) in HEX
def isOctalDigit(ch):
return (ch if isinstance(ch, unicode) else unichr(ch)) in OCTAL
return (ch if isinstance(ch, unicode) else unichr(ch)) in OCTAL
def isFutureReservedWord(w):
return w in ('enum', 'export', 'import', 'super')
RESERVED_WORD = set(('implements', 'interface', 'package', 'private', 'protected', 'public', 'static', 'yield', 'let'))
RESERVED_WORD = set(('implements', 'interface', 'package', 'private',
'protected', 'public', 'static', 'yield', 'let'))
def isStrictModeReservedWord(w):
return w in RESERVED_WORD
def isRestrictedWord(w):
return w in ('eval', 'arguments')
return w in ('eval', 'arguments')
KEYWORDS = set(
('if', 'in', 'do', 'var', 'for', 'new', 'try', 'let', 'this', 'else',
'case', 'void', 'with', 'enum', 'while', 'break', 'catch', 'throw',
'const', 'yield', 'class', 'super', 'return', 'typeof', 'delete',
'switch', 'export', 'import', 'default', 'finally', 'extends', 'function',
'continue', 'debugger', 'instanceof', 'pyimport'))
KEYWORDS = set(('if', 'in', 'do', 'var', 'for', 'new', 'try', 'let', 'this', 'else', 'case',
'void', 'with', 'enum', 'while', 'break', 'catch', 'throw', 'const', 'yield',
'class', 'super', 'return', 'typeof', 'delete', 'switch', 'export', 'import',
'default', 'finally', 'extends', 'function', 'continue', 'debugger', 'instanceof', 'pyimport'))
def isKeyword(w):
# 'const' is specialized as Keyword in V8.
# 'yield' and 'let' are for compatibility with SpiderMonkey and ES.next.
# Some others are from future reserved words.
return w in KEYWORDS
# 'const' is specialized as Keyword in V8.
# 'yield' and 'let' are for compatibility with SpiderMonkey and ES.next.
# Some others are from future reserved words.
return w in KEYWORDS
class JsSyntaxError(Exception): pass
class JsSyntaxError(Exception):
pass
if __name__=='__main__':
if __name__ == '__main__':
assert isLineTerminator('\n')
assert isLineTerminator(0x0A)
assert isIdentifierStart('$')
assert isIdentifierStart(100)
assert isWhiteSpace(' ')
assert isWhiteSpace(' ')

153
lib/pyjsparser/std_nodes.py
View File

@@ -1,6 +1,16 @@
from .pyjsparserdata import *
class Ecma51NotSupported(Exception):
def __init__(self, feature):
super(Ecma51NotSupported,
self).__init__("%s is not supported by ECMA 5.1." % feature)
self.feature = feature
def get_feature(self):
return self.feature
class BaseNode:
def finish(self):
pass
@@ -17,17 +27,6 @@ class BaseNode:
self.finish()
return self
def finishArrowFunctionExpression(self, params, defaults, body, expression):
self.type = Syntax.ArrowFunctionExpression
self.id = None
self.params = params
self.defaults = defaults
self.body = body
self.generator = False
self.expression = expression
self.finish()
return self
def finishAssignmentExpression(self, operator, left, right):
self.type = Syntax.AssignmentExpression
self.operator = operator
@@ -44,7 +43,8 @@ class BaseNode:
return self
def finishBinaryExpression(self, operator, left, right):
self.type = Syntax.LogicalExpression if (operator == '||' or operator == '&&') else Syntax.BinaryExpression
self.type = Syntax.LogicalExpression if (
operator == '||' or operator == '&&') else Syntax.BinaryExpression
self.operator = operator
self.left = left
self.right = right
@@ -77,28 +77,6 @@ class BaseNode:
self.finish()
return self
def finishClassBody(self, body):
self.type = Syntax.ClassBody
self.body = body
self.finish()
return self
def finishClassDeclaration(self, id, superClass, body):
self.type = Syntax.ClassDeclaration
self.id = id
self.superClass = superClass
self.body = body
self.finish()
return self
def finishClassExpression(self, id, superClass, body):
self.type = Syntax.ClassExpression
self.id = id
self.superClass = superClass
self.body = body
self.finish()
return self
def finishConditionalExpression(self, test, consequent, alternate):
self.type = Syntax.ConditionalExpression
self.test = test
@@ -200,7 +178,7 @@ class BaseNode:
def finishLiteral(self, token):
self.type = Syntax.Literal
self.value = token['value']
self.raw = None # todo fix it?
self.raw = token['raw']
if token.get('regex'):
self.regex = token['regex']
self.finish()
@@ -264,12 +242,6 @@ class BaseNode:
self.finish()
return self
def finishRestElement(self, argument):
self.type = Syntax.RestElement
self.argument = argument
self.finish()
return self
def finishReturnStatement(self, argument):
self.type = Syntax.ReturnStatement
self.argument = argument
@@ -282,12 +254,6 @@ class BaseNode:
self.finish()
return self
def finishSpreadElement(self, argument):
self.type = Syntax.SpreadElement
self.argument = argument
self.finish()
return self
def finishSwitchCase(self, test, consequent):
self.type = Syntax.SwitchCase
self.test = test
@@ -295,11 +261,6 @@ class BaseNode:
self.finish()
return self
def finishSuper(self, ):
self.type = Syntax.Super
self.finish()
return self
def finishSwitchStatement(self, discriminant, cases):
self.type = Syntax.SwitchStatement
self.discriminant = discriminant
@@ -307,27 +268,6 @@ class BaseNode:
self.finish()
return self
def finishTaggedTemplateExpression(self, tag, quasi):
self.type = Syntax.TaggedTemplateExpression
self.tag = tag
self.quasi = quasi
self.finish()
return self
def finishTemplateElement(self, value, tail):
self.type = Syntax.TemplateElement
self.value = value
self.tail = tail
self.finish()
return self
def finishTemplateLiteral(self, quasis, expressions):
self.type = Syntax.TemplateLiteral
self.quasis = quasis
self.expressions = expressions
self.finish()
return self
def finishThisExpression(self, ):
self.type = Syntax.ThisExpression
self.finish()
@@ -350,7 +290,8 @@ class BaseNode:
return self
def finishUnaryExpression(self, operator, argument):
self.type = Syntax.UpdateExpression if (operator == '++' or operator == '--') else Syntax.UnaryExpression
self.type = Syntax.UpdateExpression if (
operator == '++' or operator == '--') else Syntax.UnaryExpression
self.operator = operator
self.argument = argument
self.prefix = True
@@ -392,58 +333,14 @@ class BaseNode:
self.finish()
return self
def finishExportSpecifier(self, local, exported):
self.type = Syntax.ExportSpecifier
self.exported = exported or local
self.local = local
self.finish()
return self
def finishImportDefaultSpecifier(self, local):
self.type = Syntax.ImportDefaultSpecifier
self.local = local
self.finish()
return self
def finishImportNamespaceSpecifier(self, local):
self.type = Syntax.ImportNamespaceSpecifier
self.local = local
self.finish()
return self
def finishExportNamedDeclaration(self, declaration, specifiers, src):
self.type = Syntax.ExportNamedDeclaration
self.declaration = declaration
self.specifiers = specifiers
self.source = src
self.finish()
return self
def finishExportDefaultDeclaration(self, declaration):
self.type = Syntax.ExportDefaultDeclaration
self.declaration = declaration
self.finish()
return self
def finishExportAllDeclaration(self, src):
self.type = Syntax.ExportAllDeclaration
self.source = src
self.finish()
return self
def finishImportSpecifier(self, local, imported):
self.type = Syntax.ImportSpecifier
self.local = local or imported
self.imported = imported
self.finish()
return self
def finishImportDeclaration(self, specifiers, src):
self.type = Syntax.ImportDeclaration
self.specifiers = specifiers
self.source = src
self.finish()
return self
def __getattr__(self, item):
if item in self.__dict__:
return self.__dict__[item]
if item.startswith('finish'):
feature = item[6:]
raise Ecma51NotSupported(feature)
else:
raise AttributeError(item)
def __getitem__(self, item):
return getattr(self, item)
@@ -451,6 +348,10 @@ class BaseNode:
def __setitem__(self, key, value):
setattr(self, key, value)
def to_dict(self):
return node_to_dict(self)
class Node(BaseNode):
pass

34
lib/requests_toolbelt/__init__.py Normal file
View File

@@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
"""
requests-toolbelt
=================
See http://toolbelt.rtfd.org/ for documentation
:copyright: (c) 2014 by Ian Cordasco and Cory Benfield
:license: Apache v2.0, see LICENSE for more details
"""
from .adapters import SSLAdapter, SourceAddressAdapter
from .auth.guess import GuessAuth
from .multipart import (
MultipartEncoder, MultipartEncoderMonitor, MultipartDecoder,
ImproperBodyPartContentException, NonMultipartContentTypeException
)
from .streaming_iterator import StreamingIterator
from .utils.user_agent import user_agent
__title__ = 'requests-toolbelt'
__authors__ = 'Ian Cordasco, Cory Benfield'
__license__ = 'Apache v2.0'
__copyright__ = 'Copyright 2014 Ian Cordasco, Cory Benfield'
__version__ = '0.9.1'
__version_info__ = tuple(int(i) for i in __version__.split('.'))
__all__ = [
'GuessAuth', 'MultipartEncoder', 'MultipartEncoderMonitor',
'MultipartDecoder', 'SSLAdapter', 'SourceAddressAdapter',
'StreamingIterator', 'user_agent', 'ImproperBodyPartContentException',
'NonMultipartContentTypeException', '__title__', '__authors__',
'__license__', '__copyright__', '__version__', '__version_info__',
]

324
lib/requests_toolbelt/_compat.py Normal file
View File

@@ -0,0 +1,324 @@
"""Private module full of compatibility hacks.
Primarily this is for downstream redistributions of requests that unvendor
urllib3 without providing a shim.
.. warning::
This module is private. If you use it, and something breaks, you were
warned
"""
import sys
import requests
try:
from requests.packages.urllib3 import fields
from requests.packages.urllib3 import filepost
from requests.packages.urllib3 import poolmanager
except ImportError:
from urllib3 import fields
from urllib3 import filepost
from urllib3 import poolmanager
try:
from requests.packages.urllib3.connection import HTTPConnection
from requests.packages.urllib3 import connection
except ImportError:
try:
from urllib3.connection import HTTPConnection
from urllib3 import connection
except ImportError:
HTTPConnection = None
connection = None
if requests.__build__ < 0x020300:
timeout = None
else:
try:
from requests.packages.urllib3.util import timeout
except ImportError:
from urllib3.util import timeout
if requests.__build__ < 0x021000:
gaecontrib = None
else:
try:
from requests.packages.urllib3.contrib import appengine as gaecontrib
except ImportError:
from urllib3.contrib import appengine as gaecontrib
if requests.__build__ < 0x021200:
PyOpenSSLContext = None
else:
try:
from requests.packages.urllib3.contrib.pyopenssl \
import PyOpenSSLContext
except ImportError:
try:
from urllib3.contrib.pyopenssl import PyOpenSSLContext
except ImportError:
PyOpenSSLContext = None
PY3 = sys.version_info > (3, 0)
if PY3:
from collections.abc import Mapping, MutableMapping
import queue
from urllib.parse import urlencode, urljoin
else:
from collections import Mapping, MutableMapping
import Queue as queue
from urllib import urlencode
from urlparse import urljoin
try:
basestring = basestring
except NameError:
basestring = (str, bytes)
class HTTPHeaderDict(MutableMapping):
"""
:param headers:
An iterable of field-value pairs. Must not contain multiple field names
when compared case-insensitively.
:param kwargs:
Additional field-value pairs to pass in to ``dict.update``.
A ``dict`` like container for storing HTTP Headers.
Field names are stored and compared case-insensitively in compliance with
RFC 7230. Iteration provides the first case-sensitive key seen for each
case-insensitive pair.
Using ``__setitem__`` syntax overwrites fields that compare equal
case-insensitively in order to maintain ``dict``'s api. For fields that
compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
in a loop.
If multiple fields that are equal case-insensitively are passed to the
constructor or ``.update``, the behavior is undefined and some will be
lost.
>>> headers = HTTPHeaderDict()
>>> headers.add('Set-Cookie', 'foo=bar')
>>> headers.add('set-cookie', 'baz=quxx')
>>> headers['content-length'] = '7'
>>> headers['SET-cookie']
'foo=bar, baz=quxx'
>>> headers['Content-Length']
'7'
"""
def __init__(self, headers=None, **kwargs):
super(HTTPHeaderDict, self).__init__()
self._container = {}
if headers is not None:
if isinstance(headers, HTTPHeaderDict):
self._copy_from(headers)
else:
self.extend(headers)
if kwargs:
self.extend(kwargs)
def __setitem__(self, key, val):
self._container[key.lower()] = (key, val)
return self._container[key.lower()]
def __getitem__(self, key):
val = self._container[key.lower()]
return ', '.join(val[1:])
def __delitem__(self, key):
del self._container[key.lower()]
def __contains__(self, key):
return key.lower() in self._container
def __eq__(self, other):
if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
return False
if not isinstance(other, type(self)):
other = type(self)(other)
return (dict((k.lower(), v) for k, v in self.itermerged()) ==
dict((k.lower(), v) for k, v in other.itermerged()))
def __ne__(self, other):
return not self.__eq__(other)
if not PY3: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
__marker = object()
def __len__(self):
return len(self._container)
def __iter__(self):
# Only provide the originally cased names
for vals in self._container.values():
yield vals[0]
def pop(self, key, default=__marker):
"""D.pop(k[,d]) -> v, remove specified key and return its value.
If key is not found, d is returned if given, otherwise KeyError is
raised.
"""
# Using the MutableMapping function directly fails due to the private
# marker.
# Using ordinary dict.pop would expose the internal structures.
# So let's reinvent the wheel.
try:
value = self[key]
except KeyError:
if default is self.__marker:
raise
return default
else:
del self[key]
return value
def discard(self, key):
try:
del self[key]
except KeyError:
pass
def add(self, key, val):
"""Adds a (name, value) pair, doesn't overwrite the value if it already
exists.
>>> headers = HTTPHeaderDict(foo='bar')
>>> headers.add('Foo', 'baz')
>>> headers['foo']
'bar, baz'
"""
key_lower = key.lower()
new_vals = key, val
# Keep the common case aka no item present as fast as possible
vals = self._container.setdefault(key_lower, new_vals)
if new_vals is not vals:
# new_vals was not inserted, as there was a previous one
if isinstance(vals, list):
# If already several items got inserted, we have a list
vals.append(val)
else:
# vals should be a tuple then, i.e. only one item so far
# Need to convert the tuple to list for further extension
self._container[key_lower] = [vals[0], vals[1], val]
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def getlist(self, key):
"""Returns a list of all the values for the named field. Returns an
empty list if the key doesn't exist."""
try:
vals = self._container[key.lower()]
except KeyError:
return []
else:
if isinstance(vals, tuple):
return [vals[1]]
else:
return vals[1:]
# Backwards compatibility for httplib
getheaders = getlist
getallmatchingheaders = getlist
iget = getlist
def __repr__(self):
return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
def _copy_from(self, other):
for key in other:
val = other.getlist(key)
if isinstance(val, list):
# Don't need to convert tuples
val = list(val)
self._container[key.lower()] = [key] + val
def copy(self):
clone = type(self)()
clone._copy_from(self)
return clone
def iteritems(self):
"""Iterate over all header lines, including duplicate ones."""
for key in self:
vals = self._container[key.lower()]
for val in vals[1:]:
yield vals[0], val
def itermerged(self):
"""Iterate over all headers, merging duplicate ones together."""
for key in self:
val = self._container[key.lower()]
yield val[0], ', '.join(val[1:])
def items(self):
return list(self.iteritems())
@classmethod
def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
# object and extracts the multiheaders properly.
headers = []
for line in message.headers:
if line.startswith((' ', '\t')):
key, value = headers[-1]
headers[-1] = (key, value + '\r\n' + line.rstrip())
continue
key, value = line.split(':', 1)
headers.append((key, value.strip()))
return cls(headers)
__all__ = (
'basestring',
'connection',
'fields',
'filepost',
'poolmanager',
'timeout',
'HTTPHeaderDict',
'queue',
'urlencode',
'gaecontrib',
'urljoin',
'PyOpenSSLContext',
)

15
lib/requests_toolbelt/adapters/__init__.py Normal file
View File

@@ -0,0 +1,15 @@
# -*- coding: utf-8 -*-
"""
requests-toolbelt.adapters
==========================
See http://toolbelt.rtfd.org/ for documentation
:copyright: (c) 2014 by Ian Cordasco and Cory Benfield
:license: Apache v2.0, see LICENSE for more details
"""
from .ssl import SSLAdapter
from .source import SourceAddressAdapter
__all__ = ['SSLAdapter', 'SourceAddressAdapter']

206
lib/requests_toolbelt/adapters/appengine.py Normal file
View File

@@ -0,0 +1,206 @@
# -*- coding: utf-8 -*-
"""The App Engine Transport Adapter for requests.
.. versionadded:: 0.6.0
This requires a version of requests >= 2.10.0 and Python 2.
There are two ways to use this library:
#. If you're using requests directly, you can use code like:
.. code-block:: python
>>> import requests
>>> import ssl
>>> import requests.packages.urllib3.contrib.appengine as ul_appengine
>>> from requests_toolbelt.adapters import appengine
>>> s = requests.Session()
>>> if ul_appengine.is_appengine_sandbox():
... s.mount('http://', appengine.AppEngineAdapter())
... s.mount('https://', appengine.AppEngineAdapter())
#. If you depend on external libraries which use requests, you can use code
like:
.. code-block:: python
>>> from requests_toolbelt.adapters import appengine
>>> appengine.monkeypatch()
which will ensure all requests.Session objects use AppEngineAdapter properly.
You are also able to :ref:`disable certificate validation <insecure_appengine>`
when monkey-patching.
"""
import requests
import warnings
from requests import adapters
from requests import sessions
from .. import exceptions as exc
from .._compat import gaecontrib
from .._compat import timeout
class AppEngineMROHack(adapters.HTTPAdapter):
"""Resolves infinite recursion when monkeypatching.
This works by injecting itself as the base class of both the
:class:`AppEngineAdapter` and Requests' default HTTPAdapter, which needs to
be done because default HTTPAdapter's MRO is recompiled when we
monkeypatch, at which point this class becomes HTTPAdapter's base class.
In addition, we use an instantiation flag to avoid infinite recursion.
"""
_initialized = False
def __init__(self, *args, **kwargs):
if not self._initialized:
self._initialized = True
super(AppEngineMROHack, self).__init__(*args, **kwargs)
class AppEngineAdapter(AppEngineMROHack, adapters.HTTPAdapter):
"""The transport adapter for Requests to use urllib3's GAE support.
Implements Requests's HTTPAdapter API.
When deploying to Google's App Engine service, some of Requests'
functionality is broken. There is underlying support for GAE in urllib3.
This functionality, however, is opt-in and needs to be enabled explicitly
for Requests to be able to use it.
"""
__attrs__ = adapters.HTTPAdapter.__attrs__ + ['_validate_certificate']
def __init__(self, validate_certificate=True, *args, **kwargs):
_check_version()
self._validate_certificate = validate_certificate
super(AppEngineAdapter, self).__init__(*args, **kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = _AppEnginePoolManager(self._validate_certificate)
class InsecureAppEngineAdapter(AppEngineAdapter):
"""An always-insecure GAE adapter for Requests.
This is a variant of the the transport adapter for Requests to use
urllib3's GAE support that does not validate certificates. Use with
caution!
.. note::
The ``validate_certificate`` keyword argument will not be honored here
and is not part of the signature because we always force it to
``False``.
See :class:`AppEngineAdapter` for further details.
"""
def __init__(self, *args, **kwargs):
if kwargs.pop("validate_certificate", False):
warnings.warn("Certificate validation cannot be specified on the "
"InsecureAppEngineAdapter, but was present. This "
"will be ignored and certificate validation will "
"remain off.", exc.IgnoringGAECertificateValidation)
super(InsecureAppEngineAdapter, self).__init__(
validate_certificate=False, *args, **kwargs)
class _AppEnginePoolManager(object):
"""Implements urllib3's PoolManager API expected by requests.
While a real PoolManager map hostnames to reusable Connections,
AppEngine has no concept of a reusable connection to a host.
So instead, this class constructs a small Connection per request,
that is returned to the Adapter and used to access the URL.
"""
def __init__(self, validate_certificate=True):
self.appengine_manager = gaecontrib.AppEngineManager(
validate_certificate=validate_certificate)
def connection_from_url(self, url):
return _AppEngineConnection(self.appengine_manager, url)
def clear(self):
pass
class _AppEngineConnection(object):
"""Implements urllib3's HTTPConnectionPool API's urlopen().
This Connection's urlopen() is called with a host-relative path,
so in order to properly support opening the URL, we need to store
the full URL when this Connection is constructed from the PoolManager.
This code wraps AppEngineManager.urlopen(), which exposes a different
API than in the original urllib3 urlopen(), and thus needs this adapter.
"""
def __init__(self, appengine_manager, url):
self.appengine_manager = appengine_manager
self.url = url
def urlopen(self, method, url, body=None, headers=None, retries=None,
redirect=True, assert_same_host=True,
timeout=timeout.Timeout.DEFAULT_TIMEOUT,
pool_timeout=None, release_conn=None, **response_kw):
# This function's url argument is a host-relative URL,
# but the AppEngineManager expects an absolute URL.
# So we saved out the self.url when the AppEngineConnection
# was constructed, which we then can use down below instead.
# We once tried to verify our assumptions here, but sometimes the
# passed-in URL differs on url fragments, or "http://a.com" vs "/".
# urllib3's App Engine adapter only uses Timeout.total, not read or
# connect.
if not timeout.total:
timeout.total = timeout._read or timeout._connect
# Jump through the hoops necessary to call AppEngineManager's API.
return self.appengine_manager.urlopen(
method,
self.url,
body=body,
headers=headers,
retries=retries,
redirect=redirect,
timeout=timeout,
**response_kw)
def monkeypatch(validate_certificate=True):
"""Sets up all Sessions to use AppEngineAdapter by default.
If you don't want to deal with configuring your own Sessions,
or if you use libraries that use requests directly (ie requests.post),
then you may prefer to monkeypatch and auto-configure all Sessions.
.. warning: :
If ``validate_certificate`` is ``False``, certification validation will
effectively be disabled for all requests.
"""
_check_version()
# HACK: We should consider modifying urllib3 to support this cleanly,
# so that we can set a module-level variable in the sessions module,
# instead of overriding an imported HTTPAdapter as is done here.
adapter = AppEngineAdapter
if not validate_certificate:
adapter = InsecureAppEngineAdapter
sessions.HTTPAdapter = adapter
adapters.HTTPAdapter = adapter
def _check_version():
if gaecontrib is None:
raise exc.VersionMismatchError(
"The toolbelt requires at least Requests 2.10.0 to be "
"installed. Version {0} was found instead.".format(
requests.__version__
)
)

48
lib/requests_toolbelt/adapters/fingerprint.py Normal file
View File

@@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
"""Submodule containing the implementation for the FingerprintAdapter.
This file contains an implementation of a Transport Adapter that validates
the fingerprints of SSL certificates presented upon connection.
"""
from requests.adapters import HTTPAdapter
from .._compat import poolmanager
class FingerprintAdapter(HTTPAdapter):
"""
A HTTPS Adapter for Python Requests that verifies certificate fingerprints,
instead of certificate hostnames.
Example usage:
.. code-block:: python
import requests
import ssl
from requests_toolbelt.adapters.fingerprint import FingerprintAdapter
twitter_fingerprint = '...'
s = requests.Session()
s.mount(
'https://twitter.com',
FingerprintAdapter(twitter_fingerprint)
)
The fingerprint should be provided as a hexadecimal string, optionally
containing colons.
"""
__attrs__ = HTTPAdapter.__attrs__ + ['fingerprint']
def __init__(self, fingerprint, **kwargs):
self.fingerprint = fingerprint
super(FingerprintAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
assert_fingerprint=self.fingerprint)

43
lib/requests_toolbelt/adapters/host_header_ssl.py Normal file
View File

@@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.adapters.host_header_ssl
==========================================
This file contains an implementation of the HostHeaderSSLAdapter.
"""
from requests.adapters import HTTPAdapter
class HostHeaderSSLAdapter(HTTPAdapter):
"""
A HTTPS Adapter for Python Requests that sets the hostname for certificate
verification based on the Host header.
This allows requesting the IP address directly via HTTPS without getting
a "hostname doesn't match" exception.
Example usage:
>>> s.mount('https://', HostHeaderSSLAdapter())
>>> s.get("https://93.184.216.34", headers={"Host": "example.org"})
"""
def send(self, request, **kwargs):
# HTTP headers are case-insensitive (RFC 7230)
host_header = None
for header in request.headers:
if header.lower() == "host":
host_header = request.headers[header]
break
connection_pool_kwargs = self.poolmanager.connection_pool_kw
if host_header:
connection_pool_kwargs["assert_hostname"] = host_header
elif "assert_hostname" in connection_pool_kwargs:
# an assert_hostname from a previous request may have been left
connection_pool_kwargs.pop("assert_hostname", None)
return super(HostHeaderSSLAdapter, self).send(request, **kwargs)

129
lib/requests_toolbelt/adapters/socket_options.py Normal file
View File

@@ -0,0 +1,129 @@
# -*- coding: utf-8 -*-
"""The implementation of the SocketOptionsAdapter."""
import socket
import warnings
import sys
import requests
from requests import adapters
from .._compat import connection
from .._compat import poolmanager
from .. import exceptions as exc
class SocketOptionsAdapter(adapters.HTTPAdapter):
"""An adapter for requests that allows users to specify socket options.
Since version 2.4.0 of requests, it is possible to specify a custom list
of socket options that need to be set before establishing the connection.
Example usage::
>>> import socket
>>> import requests
>>> from requests_toolbelt.adapters import socket_options
>>> s = requests.Session()
>>> opts = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0)]
>>> adapter = socket_options.SocketOptionsAdapter(socket_options=opts)
>>> s.mount('http://', adapter)
You can also take advantage of the list of default options on this class
to keep using the original options in addition to your custom options. In
that case, ``opts`` might look like::
>>> opts = socket_options.SocketOptionsAdapter.default_options + opts
"""
if connection is not None:
default_options = getattr(
connection.HTTPConnection,
'default_socket_options',
[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
)
else:
default_options = []
warnings.warn(exc.RequestsVersionTooOld,
"This version of Requests is only compatible with a "
"version of urllib3 which is too old to support "
"setting options on a socket. This adapter is "
"functionally useless.")
def __init__(self, **kwargs):
self.socket_options = kwargs.pop('socket_options',
self.default_options)
super(SocketOptionsAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
if requests.__build__ >= 0x020400:
# NOTE(Ian): Perhaps we should raise a warning
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
socket_options=self.socket_options
)
else:
super(SocketOptionsAdapter, self).init_poolmanager(
connections, maxsize, block
)
class TCPKeepAliveAdapter(SocketOptionsAdapter):
"""An adapter for requests that turns on TCP Keep-Alive by default.
The adapter sets 4 socket options:
- ``SOL_SOCKET`` ``SO_KEEPALIVE`` - This turns on TCP Keep-Alive
- ``IPPROTO_TCP`` ``TCP_KEEPINTVL`` 20 - Sets the keep alive interval
- ``IPPROTO_TCP`` ``TCP_KEEPCNT`` 5 - Sets the number of keep alive probes
- ``IPPROTO_TCP`` ``TCP_KEEPIDLE`` 60 - Sets the keep alive time if the
socket library has the ``TCP_KEEPIDLE`` constant
The latter three can be overridden by keyword arguments (respectively):
- ``idle``
- ``interval``
- ``count``
You can use this adapter like so::
>>> from requests_toolbelt.adapters import socket_options
>>> tcp = socket_options.TCPKeepAliveAdapter(idle=120, interval=10)
>>> s = requests.Session()
>>> s.mount('http://', tcp)
"""
def __init__(self, **kwargs):
socket_options = kwargs.pop('socket_options',
SocketOptionsAdapter.default_options)
idle = kwargs.pop('idle', 60)
interval = kwargs.pop('interval', 20)
count = kwargs.pop('count', 5)
socket_options = socket_options + [
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
]
# NOTE(Ian): OSX does not have these constants defined, so we
# set them conditionally.
if getattr(socket, 'TCP_KEEPINTVL', None) is not None:
socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
interval)]
elif sys.platform == 'darwin':
# On OSX, TCP_KEEPALIVE from netinet/tcp.h is not exported
# by python's socket module
TCP_KEEPALIVE = getattr(socket, 'TCP_KEEPALIVE', 0x10)
socket_options += [(socket.IPPROTO_TCP, TCP_KEEPALIVE, interval)]
if getattr(socket, 'TCP_KEEPCNT', None) is not None:
socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, count)]
if getattr(socket, 'TCP_KEEPIDLE', None) is not None:
socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle)]
super(TCPKeepAliveAdapter, self).__init__(
socket_options=socket_options, **kwargs
)

67
lib/requests_toolbelt/adapters/source.py Normal file
View File

@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.source_adapter
================================
This file contains an implementation of the SourceAddressAdapter originally
demonstrated on the Requests GitHub page.
"""
from requests.adapters import HTTPAdapter
from .._compat import poolmanager, basestring
class SourceAddressAdapter(HTTPAdapter):
"""
A Source Address Adapter for Python Requests that enables you to choose the
local address to bind to. This allows you to send your HTTP requests from a
specific interface and IP address.
Two address formats are accepted. The first is a string: this will set the
local IP address to the address given in the string, and will also choose a
semi-random high port for the local port number.
The second is a two-tuple of the form (ip address, port): for example,
``('10.10.10.10', 8999)``. This will set the local IP address to the first
element, and the local port to the second element. If ``0`` is used as the
port number, a semi-random high port will be selected.
.. warning:: Setting an explicit local port can have negative interactions
with connection-pooling in Requests: in particular, it risks
the possibility of getting "Address in use" errors. The
string-only argument is generally preferred to the tuple-form.
Example usage:
.. code-block:: python
import requests
from requests_toolbelt.adapters.source import SourceAddressAdapter
s = requests.Session()
s.mount('http://', SourceAddressAdapter('10.10.10.10'))
s.mount('https://', SourceAddressAdapter(('10.10.10.10', 8999)))
"""
def __init__(self, source_address, **kwargs):
if isinstance(source_address, basestring):
self.source_address = (source_address, 0)
elif isinstance(source_address, tuple):
self.source_address = source_address
else:
raise TypeError(
"source_address must be IP address string or (ip, port) tuple"
)
super(SourceAddressAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
source_address=self.source_address)
def proxy_manager_for(self, *args, **kwargs):
kwargs['source_address'] = self.source_address
return super(SourceAddressAdapter, self).proxy_manager_for(
*args, **kwargs)

66
lib/requests_toolbelt/adapters/ssl.py Normal file
View File

@@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.ssl_adapter
=============================
This file contains an implementation of the SSLAdapter originally demonstrated
in this blog post:
https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/
"""
import requests
from requests.adapters import HTTPAdapter
from .._compat import poolmanager
class SSLAdapter(HTTPAdapter):
"""
A HTTPS Adapter for Python Requests that allows the choice of the SSL/TLS
version negotiated by Requests. This can be used either to enforce the
choice of high-security TLS versions (where supported), or to work around
misbehaving servers that fail to correctly negotiate the default TLS
version being offered.
Example usage:
>>> import requests
>>> import ssl
>>> from requests_toolbelt import SSLAdapter
>>> s = requests.Session()
>>> s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
You can replace the chosen protocol with any that are available in the
default Python SSL module. All subsequent requests that match the adapter
prefix will use the chosen SSL version instead of the default.
This adapter will also attempt to change the SSL/TLS version negotiated by
Requests when using a proxy. However, this may not always be possible:
prior to Requests v2.4.0 the adapter did not have access to the proxy setup
code. In earlier versions of Requests, this adapter will not function
properly when used with proxies.
"""
__attrs__ = HTTPAdapter.__attrs__ + ['ssl_version']
def __init__(self, ssl_version=None, **kwargs):
self.ssl_version = ssl_version
super(SSLAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=self.ssl_version)
if requests.__build__ >= 0x020400:
# Earlier versions of requests either don't have this method or, worse,
# don't allow passing arbitrary keyword arguments. As a result, only
# conditionally define this method.
def proxy_manager_for(self, *args, **kwargs):
kwargs['ssl_version'] = self.ssl_version
return super(SSLAdapter, self).proxy_manager_for(*args, **kwargs)

178
lib/requests_toolbelt/adapters/x509.py Normal file
View File

@@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
"""A X509Adapter for use with the requests library.
This file contains an implementation of the X509Adapter that will
allow users to authenticate a request using an arbitrary
X.509 certificate without needing to convert it to a .pem file
"""
from OpenSSL.crypto import PKey, X509
from cryptography import x509
from cryptography.hazmat.primitives.serialization import (load_pem_private_key,
load_der_private_key)
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.backends import default_backend
from datetime import datetime
from requests.adapters import HTTPAdapter
import requests
from .._compat import PyOpenSSLContext
from .. import exceptions as exc
"""
importing the protocol constants from _ssl instead of ssl because only the
constants are needed and to handle issues caused by importing from ssl on
the 2.7.x line.
"""
try:
from _ssl import PROTOCOL_TLS as PROTOCOL
except ImportError:
from _ssl import PROTOCOL_SSLv23 as PROTOCOL
class X509Adapter(HTTPAdapter):
r"""Adapter for use with X.509 certificates.
Provides an interface for Requests sessions to contact HTTPS urls and
authenticate with an X.509 cert by implementing the Transport Adapter
interface. This class will need to be manually instantiated and mounted
to the session
:param pool_connections: The number of urllib3 connection pools to
cache.
:param pool_maxsize: The maximum number of connections to save in the
pool.
:param max_retries: The maximum number of retries each connection
should attempt. Note, this applies only to failed DNS lookups,
socket connections and connection timeouts, never to requests where
data has made it to the server. By default, Requests does not retry
failed connections. If you need granular control over the
conditions under which we retry a request, import urllib3's
``Retry`` class and pass that instead.
:param pool_block: Whether the connection pool should block for
connections.
:param bytes cert_bytes:
bytes object containing contents of a cryptography.x509Certificate
object using the encoding specified by the ``encoding`` parameter.
:param bytes pk_bytes:
bytes object containing contents of a object that implements
``cryptography.hazmat.primitives.serialization.PrivateFormat``
using the encoding specified by the ``encoding`` parameter.
:param password:
string or utf8 encoded bytes containing the passphrase used for the
private key. None if unencrypted. Defaults to None.
:param encoding:
Enumeration detailing the encoding method used on the ``cert_bytes``
parameter. Can be either PEM or DER. Defaults to PEM.
:type encoding:
:class: `cryptography.hazmat.primitives.serialization.Encoding`
Usage::
>>> import requests
>>> from requests_toolbelt.adapters.x509 import X509Adapter
>>> s = requests.Session()
>>> a = X509Adapter(max_retries=3,
cert_bytes=b'...', pk_bytes=b'...', encoding='...'
>>> s.mount('https://', a)
"""
def __init__(self, *args, **kwargs):
self._check_version()
cert_bytes = kwargs.pop('cert_bytes', None)
pk_bytes = kwargs.pop('pk_bytes', None)
password = kwargs.pop('password', None)
encoding = kwargs.pop('encoding', Encoding.PEM)
password_bytes = None
if cert_bytes is None or not isinstance(cert_bytes, bytes):
raise ValueError('Invalid cert content provided. '
'You must provide an X.509 cert '
'formatted as a byte array.')
if pk_bytes is None or not isinstance(pk_bytes, bytes):
raise ValueError('Invalid private key content provided. '
'You must provide a private key '
'formatted as a byte array.')
if isinstance(password, bytes):
password_bytes = password
elif password:
password_bytes = password.encode('utf8')
self.ssl_context = create_ssl_context(cert_bytes, pk_bytes,
password_bytes, encoding)
super(X509Adapter, self).__init__(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
if self.ssl_context:
kwargs['ssl_context'] = self.ssl_context
return super(X509Adapter, self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
if self.ssl_context:
kwargs['ssl_context'] = self.ssl_context
return super(X509Adapter, self).proxy_manager_for(*args, **kwargs)
def _check_version(self):
if PyOpenSSLContext is None:
raise exc.VersionMismatchError(
"The X509Adapter requires at least Requests 2.12.0 to be "
"installed. Version {0} was found instead.".format(
requests.__version__
)
)
def check_cert_dates(cert):
"""Verify that the supplied client cert is not invalid."""
now = datetime.utcnow()
if cert.not_valid_after < now or cert.not_valid_before > now:
raise ValueError('Client certificate expired: Not After: '
'{0:%Y-%m-%d %H:%M:%SZ} '
'Not Before: {1:%Y-%m-%d %H:%M:%SZ}'
.format(cert.not_valid_after, cert.not_valid_before))
def create_ssl_context(cert_byes, pk_bytes, password=None,
encoding=Encoding.PEM):
"""Create an SSL Context with the supplied cert/password.
:param cert_bytes array of bytes containing the cert encoded
using the method supplied in the ``encoding`` parameter
:param pk_bytes array of bytes containing the private key encoded
using the method supplied in the ``encoding`` parameter
:param password array of bytes containing the passphrase to be used
with the supplied private key. None if unencrypted.
Defaults to None.
:param encoding ``cryptography.hazmat.primitives.serialization.Encoding``
details the encoding method used on the ``cert_bytes`` and
``pk_bytes`` parameters. Can be either PEM or DER.
Defaults to PEM.
"""
backend = default_backend()
cert = None
key = None
if encoding == Encoding.PEM:
cert = x509.load_pem_x509_certificate(cert_byes, backend)
key = load_pem_private_key(pk_bytes, password, backend)
elif encoding == Encoding.DER:
cert = x509.load_der_x509_certificate(cert_byes, backend)
key = load_der_private_key(pk_bytes, password, backend)
else:
raise ValueError('Invalid encoding provided: Must be PEM or DER')
if not (cert and key):
raise ValueError('Cert and key could not be parsed from '
'provided data')
check_cert_dates(cert)
ssl_context = PyOpenSSLContext(PROTOCOL)
ssl_context._ctx.use_certificate(X509.from_cryptography(cert))
ssl_context._ctx.use_privatekey(PKey.from_cryptography_key(key))
return ssl_context

0
lib/requests_toolbelt/auth/__init__.py Normal file
View File

29
lib/requests_toolbelt/auth/_digest_auth_compat.py Normal file
View File

@@ -0,0 +1,29 @@
"""Provide a compatibility layer for requests.auth.HTTPDigestAuth."""
import requests
class _ThreadingDescriptor(object):
def __init__(self, prop, default):
self.prop = prop
self.default = default
def __get__(self, obj, objtype=None):
return getattr(obj._thread_local, self.prop, self.default)
def __set__(self, obj, value):
setattr(obj._thread_local, self.prop, value)
class _HTTPDigestAuth(requests.auth.HTTPDigestAuth):
init = _ThreadingDescriptor('init', True)
last_nonce = _ThreadingDescriptor('last_nonce', '')
nonce_count = _ThreadingDescriptor('nonce_count', 0)
chal = _ThreadingDescriptor('chal', {})
pos = _ThreadingDescriptor('pos', None)
num_401_calls = _ThreadingDescriptor('num_401_calls', 1)
if requests.__build__ < 0x020800:
HTTPDigestAuth = requests.auth.HTTPDigestAuth
else:
HTTPDigestAuth = _HTTPDigestAuth

146
lib/requests_toolbelt/auth/guess.py Normal file
View File

@@ -0,0 +1,146 @@
# -*- coding: utf-8 -*-
"""The module containing the code for GuessAuth."""
from requests import auth
from requests import cookies
from . import _digest_auth_compat as auth_compat, http_proxy_digest
class GuessAuth(auth.AuthBase):
"""Guesses the auth type by the WWW-Authentication header."""
def __init__(self, username, password):
self.username = username
self.password = password
self.auth = None
self.pos = None
def _handle_basic_auth_401(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.auth = auth.HTTPBasicAuth(self.username, self.password)
prep = self.auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_digest_auth_401(self, r, kwargs):
self.auth = auth_compat.HTTPDigestAuth(self.username, self.password)
try:
self.auth.init_per_thread_state()
except AttributeError:
# If we're not on requests 2.8.0+ this method does not exist and
# is not relevant.
pass
# Check that the attr exists because much older versions of requests
# set this attribute lazily. For example:
# https://github.com/kennethreitz/requests/blob/33735480f77891754304e7f13e3cdf83aaaa76aa/requests/auth.py#L59
if (hasattr(self.auth, 'num_401_calls') and
self.auth.num_401_calls is None):
self.auth.num_401_calls = 1
# Digest auth would resend the request by itself. We can take a
# shortcut here.
return self.auth.handle_401(r, **kwargs)
def handle_401(self, r, **kwargs):
"""Resends a request with auth headers, if needed."""
www_authenticate = r.headers.get('www-authenticate', '').lower()
if 'basic' in www_authenticate:
return self._handle_basic_auth_401(r, kwargs)
if 'digest' in www_authenticate:
return self._handle_digest_auth_401(r, kwargs)
def __call__(self, request):
if self.auth is not None:
return self.auth(request)
try:
self.pos = request.body.tell()
except AttributeError:
pass
request.register_hook('response', self.handle_401)
return request
class GuessProxyAuth(GuessAuth):
"""
Guesses the auth type by WWW-Authentication and Proxy-Authentication
headers
"""
def __init__(self, username=None, password=None,
proxy_username=None, proxy_password=None):
super(GuessProxyAuth, self).__init__(username, password)
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.proxy_auth = None
def _handle_basic_auth_407(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username,
self.proxy_password)
prep = self.proxy_auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_digest_auth_407(self, r, kwargs):
self.proxy_auth = http_proxy_digest.HTTPProxyDigestAuth(
username=self.proxy_username,
password=self.proxy_password)
try:
self.auth.init_per_thread_state()
except AttributeError:
pass
return self.proxy_auth.handle_407(r, **kwargs)
def handle_407(self, r, **kwargs):
proxy_authenticate = r.headers.get('Proxy-Authenticate', '').lower()
if 'basic' in proxy_authenticate:
return self._handle_basic_auth_407(r, kwargs)
if 'digest' in proxy_authenticate:
return self._handle_digest_auth_407(r, kwargs)
def __call__(self, request):
if self.proxy_auth is not None:
request = self.proxy_auth(request)
try:
self.pos = request.body.tell()
except AttributeError:
pass
request.register_hook('response', self.handle_407)
return super(GuessProxyAuth, self).__call__(request)

142
lib/requests_toolbelt/auth/handler.py Normal file
View File

@@ -0,0 +1,142 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.auth.handler
==============================
This holds all of the implementation details of the Authentication Handler.
"""
from requests.auth import AuthBase, HTTPBasicAuth
from requests.compat import urlparse, urlunparse
class AuthHandler(AuthBase):
"""
The ``AuthHandler`` object takes a dictionary of domains paired with
authentication strategies and will use this to determine which credentials
to use when making a request. For example, you could do the following:
.. code-block:: python
from requests import HTTPDigestAuth
from requests_toolbelt.auth.handler import AuthHandler
import requests
auth = AuthHandler({
'https://api.github.com': ('sigmavirus24', 'fakepassword'),
'https://example.com': HTTPDigestAuth('username', 'password')
})
r = requests.get('https://api.github.com/user', auth=auth)
# => <Response [200]>
r = requests.get('https://example.com/some/path', auth=auth)
# => <Response [200]>
s = requests.Session()
s.auth = auth
r = s.get('https://api.github.com/user')
# => <Response [200]>
.. warning::
:class:`requests.auth.HTTPDigestAuth` is not yet thread-safe. If you
use :class:`AuthHandler` across multiple threads you should
instantiate a new AuthHandler for each thread with a new
HTTPDigestAuth instance for each thread.
"""
def __init__(self, strategies):
self.strategies = dict(strategies)
self._make_uniform()
def __call__(self, request):
auth = self.get_strategy_for(request.url)
return auth(request)
def __repr__(self):
return '<AuthHandler({0!r})>'.format(self.strategies)
def _make_uniform(self):
existing_strategies = list(self.strategies.items())
self.strategies = {}
for (k, v) in existing_strategies:
self.add_strategy(k, v)
@staticmethod
def _key_from_url(url):
parsed = urlparse(url)
return urlunparse((parsed.scheme.lower(),
parsed.netloc.lower(),
'', '', '', ''))
def add_strategy(self, domain, strategy):
"""Add a new domain and authentication strategy.
:param str domain: The domain you wish to match against. For example:
``'https://api.github.com'``
:param str strategy: The authentication strategy you wish to use for
that domain. For example: ``('username', 'password')`` or
``requests.HTTPDigestAuth('username', 'password')``
.. code-block:: python
a = AuthHandler({})
a.add_strategy('https://api.github.com', ('username', 'password'))
"""
# Turn tuples into Basic Authentication objects
if isinstance(strategy, tuple):
strategy = HTTPBasicAuth(*strategy)
key = self._key_from_url(domain)
self.strategies[key] = strategy
def get_strategy_for(self, url):
"""Retrieve the authentication strategy for a specified URL.
:param str url: The full URL you will be making a request against. For
example, ``'https://api.github.com/user'``
:returns: Callable that adds authentication to a request.
.. code-block:: python
import requests
a = AuthHandler({'example.com', ('foo', 'bar')})
strategy = a.get_strategy_for('http://example.com/example')
assert isinstance(strategy, requests.auth.HTTPBasicAuth)
"""
key = self._key_from_url(url)
return self.strategies.get(key, NullAuthStrategy())
def remove_strategy(self, domain):
"""Remove the domain and strategy from the collection of strategies.
:param str domain: The domain you wish remove. For example,
``'https://api.github.com'``.
.. code-block:: python
a = AuthHandler({'example.com', ('foo', 'bar')})
a.remove_strategy('example.com')
assert a.strategies == {}
"""
key = self._key_from_url(domain)
if key in self.strategies:
del self.strategies[key]
class NullAuthStrategy(AuthBase):
def __repr__(self):
return '<NullAuthStrategy>'
def __call__(self, r):
return r

103
lib/requests_toolbelt/auth/http_proxy_digest.py Normal file
View File

@@ -0,0 +1,103 @@
# -*- coding: utf-8 -*-
"""The module containing HTTPProxyDigestAuth."""
import re
from requests import cookies, utils
from . import _digest_auth_compat as auth
class HTTPProxyDigestAuth(auth.HTTPDigestAuth):
"""HTTP digest authentication between proxy
:param stale_rejects: The number of rejects indicate that:
the client may wish to simply retry the request
with a new encrypted response, without reprompting the user for a
new username and password. i.e., retry build_digest_header
:type stale_rejects: int
"""
_pat = re.compile(r'digest ', flags=re.IGNORECASE)
def __init__(self, *args, **kwargs):
super(HTTPProxyDigestAuth, self).__init__(*args, **kwargs)
self.stale_rejects = 0
self.init_per_thread_state()
@property
def stale_rejects(self):
thread_local = getattr(self, '_thread_local', None)
if thread_local is None:
return self._stale_rejects
return thread_local.stale_rejects
@stale_rejects.setter
def stale_rejects(self, value):
thread_local = getattr(self, '_thread_local', None)
if thread_local is None:
self._stale_rejects = value
else:
thread_local.stale_rejects = value
def init_per_thread_state(self):
try:
super(HTTPProxyDigestAuth, self).init_per_thread_state()
except AttributeError:
# If we're not on requests 2.8.0+ this method does not exist
pass
def handle_407(self, r, **kwargs):
"""Handle HTTP 407 only once, otherwise give up
:param r: current response
:returns: responses, along with the new response
"""
if r.status_code == 407 and self.stale_rejects < 2:
s_auth = r.headers.get("proxy-authenticate")
if s_auth is None:
raise IOError(
"proxy server violated RFC 7235:"
"407 response MUST contain header proxy-authenticate")
elif not self._pat.match(s_auth):
return r
self.chal = utils.parse_dict_header(
self._pat.sub('', s_auth, count=1))
# if we present the user/passwd and still get rejected
# http://tools.ietf.org/html/rfc2617#section-3.2.1
if ('Proxy-Authorization' in r.request.headers and
'stale' in self.chal):
if self.chal['stale'].lower() == 'true': # try again
self.stale_rejects += 1
# wrong user/passwd
elif self.chal['stale'].lower() == 'false':
raise IOError("User or password is invalid")
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
else: # give up authenticate
return r
def __call__(self, r):
self.init_per_thread_state()
# if we have nonce, then just use it, otherwise server will tell us
if self.last_nonce:
r.headers['Proxy-Authorization'] = self.build_digest_header(
r.method, r.url
)
r.register_hook('response', self.handle_407)
return r

0
lib/requests_toolbelt/cookies/__init__.py Normal file
View File

7
lib/requests_toolbelt/cookies/forgetful.py Normal file
View File

@@ -0,0 +1,7 @@
"""The module containing the code for ForgetfulCookieJar."""
from requests.cookies import RequestsCookieJar
class ForgetfulCookieJar(RequestsCookieJar):
def set_cookie(self, *args, **kwargs):
return

0
lib/requests_toolbelt/downloadutils/__init__.py Normal file
View File

177
lib/requests_toolbelt/downloadutils/stream.py Normal file
View File

@@ -0,0 +1,177 @@
# -*- coding: utf-8 -*-
"""Utilities for dealing with streamed requests."""
import collections
import os.path
import re
from .. import exceptions as exc
# Regular expressions stolen from werkzeug/http.py
# cd2c97bb0a076da2322f11adce0b2731f9193396 L62-L64
_QUOTED_STRING_RE = r'"[^"\\]*(?:\\.[^"\\]*)*"'
_OPTION_HEADER_PIECE_RE = re.compile(
r';\s*(%s|[^\s;=]+)\s*(?:=\s*(%s|[^;]+))?\s*' % (_QUOTED_STRING_RE,
_QUOTED_STRING_RE)
)
_DEFAULT_CHUNKSIZE = 512
def _get_filename(content_disposition):
for match in _OPTION_HEADER_PIECE_RE.finditer(content_disposition):
k, v = match.groups()
if k == 'filename':
# ignore any directory paths in the filename
return os.path.split(v)[1]
return None
def get_download_file_path(response, path):
"""
Given a response and a path, return a file path for a download.
If a ``path`` parameter is a directory, this function will parse the
``Content-Disposition`` header on the response to determine the name of the
file as reported by the server, and return a file path in the specified
directory.
If ``path`` is empty or None, this function will return a path relative
to the process' current working directory.
If path is a full file path, return it.
:param response: A Response object from requests
:type response: requests.models.Response
:param str path: Directory or file path.
:returns: full file path to download as
:rtype: str
:raises: :class:`requests_toolbelt.exceptions.StreamingError`
"""
path_is_dir = path and os.path.isdir(path)
if path and not path_is_dir:
# fully qualified file path
filepath = path
else:
response_filename = _get_filename(
response.headers.get('content-disposition', '')
)
if not response_filename:
raise exc.StreamingError('No filename given to stream response to')
if path_is_dir:
# directory to download to
filepath = os.path.join(path, response_filename)
else:
# fallback to downloading to current working directory
filepath = response_filename
return filepath
def stream_response_to_file(response, path=None, chunksize=_DEFAULT_CHUNKSIZE):
"""Stream a response body to the specified file.
Either use the ``path`` provided or use the name provided in the
``Content-Disposition`` header.
.. warning::
If you pass this function an open file-like object as the ``path``
parameter, the function will not close that file for you.
.. warning::
This function will not automatically close the response object
passed in as the ``response`` parameter.
If a ``path`` parameter is a directory, this function will parse the
``Content-Disposition`` header on the response to determine the name of the
file as reported by the server, and return a file path in the specified
directory. If no ``path`` parameter is supplied, this function will default
to the process' current working directory.
.. code-block:: python
import requests
from requests_toolbelt import exceptions
from requests_toolbelt.downloadutils import stream
r = requests.get(url, stream=True)
try:
filename = stream.stream_response_to_file(r)
except exceptions.StreamingError as e:
# The toolbelt could not find the filename in the
# Content-Disposition
print(e.message)
You can also specify the filename as a string. This will be passed to
the built-in :func:`open` and we will read the content into the file.
.. code-block:: python
import requests
from requests_toolbelt.downloadutils import stream
r = requests.get(url, stream=True)
filename = stream.stream_response_to_file(r, path='myfile')
If the calculated download file path already exists, this function will
raise a StreamingError.
Instead, if you want to manage the file object yourself, you need to
provide either a :class:`io.BytesIO` object or a file opened with the
`'b'` flag. See the two examples below for more details.
.. code-block:: python
import requests
from requests_toolbelt.downloadutils import stream
with open('myfile', 'wb') as fd:
r = requests.get(url, stream=True)
filename = stream.stream_response_to_file(r, path=fd)
print('{0} saved to {1}'.format(url, filename))
.. code-block:: python
import io
import requests
from requests_toolbelt.downloadutils import stream
b = io.BytesIO()
r = requests.get(url, stream=True)
filename = stream.stream_response_to_file(r, path=b)
assert filename is None
:param response: A Response object from requests
:type response: requests.models.Response
:param path: *(optional)*, Either a string with the path to the location
to save the response content, or a file-like object expecting bytes.
:type path: :class:`str`, or object with a :meth:`write`
:param int chunksize: (optional), Size of chunk to attempt to stream
(default 512B).
:returns: The name of the file, if one can be determined, else None
:rtype: str
:raises: :class:`requests_toolbelt.exceptions.StreamingError`
"""
pre_opened = False
fd = None
filename = None
if path and isinstance(getattr(path, 'write', None), collections.Callable):
pre_opened = True
fd = path
filename = getattr(fd, 'name', None)
else:
filename = get_download_file_path(response, path)
if os.path.exists(filename):
raise exc.StreamingError("File already exists: %s" % filename)
fd = open(filename, 'wb')
for chunk in response.iter_content(chunk_size=chunksize):
fd.write(chunk)
if not pre_opened:
fd.close()
return filename

123
lib/requests_toolbelt/downloadutils/tee.py Normal file
View File

@@ -0,0 +1,123 @@
"""Tee function implementations."""
import io
_DEFAULT_CHUNKSIZE = 65536
__all__ = ['tee', 'tee_to_file', 'tee_to_bytearray']
def _tee(response, callback, chunksize, decode_content):
for chunk in response.raw.stream(amt=chunksize,
decode_content=decode_content):
callback(chunk)
yield chunk
def tee(response, fileobject, chunksize=_DEFAULT_CHUNKSIZE,
decode_content=None):
"""Stream the response both to the generator and a file.
This will stream the response body while writing the bytes to
``fileobject``.
Example usage:
.. code-block:: python
resp = requests.get(url, stream=True)
with open('save_file', 'wb') as save_file:
for chunk in tee(resp, save_file):
# do stuff with chunk
.. code-block:: python
import io
resp = requests.get(url, stream=True)
fileobject = io.BytesIO()
for chunk in tee(resp, fileobject):
# do stuff with chunk
:param response: Response from requests.
:type response: requests.Response
:param fileobject: Writable file-like object.
:type fileobject: file, io.BytesIO
:param int chunksize: (optional), Size of chunk to attempt to stream.
:param bool decode_content: (optional), If True, this will decode the
compressed content of the response.
:raises: TypeError if the fileobject wasn't opened with the right mode
or isn't a BytesIO object.
"""
# We will be streaming the raw bytes from over the wire, so we need to
# ensure that writing to the fileobject will preserve those bytes. On
# Python3, if the user passes an io.StringIO, this will fail, so we need
# to check for BytesIO instead.
if not ('b' in getattr(fileobject, 'mode', '') or
isinstance(fileobject, io.BytesIO)):
raise TypeError('tee() will write bytes directly to this fileobject'
', it must be opened with the "b" flag if it is a file'
' or inherit from io.BytesIO.')
return _tee(response, fileobject.write, chunksize, decode_content)
def tee_to_file(response, filename, chunksize=_DEFAULT_CHUNKSIZE,
decode_content=None):
"""Stream the response both to the generator and a file.
This will open a file named ``filename`` and stream the response body
while writing the bytes to the opened file object.
Example usage:
.. code-block:: python
resp = requests.get(url, stream=True)
for chunk in tee_to_file(resp, 'save_file'):
# do stuff with chunk
:param response: Response from requests.
:type response: requests.Response
:param str filename: Name of file in which we write the response content.
:param int chunksize: (optional), Size of chunk to attempt to stream.
:param bool decode_content: (optional), If True, this will decode the
compressed content of the response.
"""
with open(filename, 'wb') as fd:
for chunk in tee(response, fd, chunksize, decode_content):
yield chunk
def tee_to_bytearray(response, bytearr, chunksize=_DEFAULT_CHUNKSIZE,
decode_content=None):
"""Stream the response both to the generator and a bytearray.
This will stream the response provided to the function, add them to the
provided :class:`bytearray` and yield them to the user.
.. note::
This uses the :meth:`bytearray.extend` by default instead of passing
the bytearray into the ``readinto`` method.
Example usage:
.. code-block:: python
b = bytearray()
resp = requests.get(url, stream=True)
for chunk in tee_to_bytearray(resp, b):
# do stuff with chunk
:param response: Response from requests.
:type response: requests.Response
:param bytearray bytearr: Array to add the streamed bytes to.
:param int chunksize: (optional), Size of chunk to attempt to stream.
:param bool decode_content: (optional), If True, this will decode the
compressed content of the response.
"""
if not isinstance(bytearr, bytearray):
raise TypeError('tee_to_bytearray() expects bytearr to be a '
'bytearray')
return _tee(response, bytearr.extend, chunksize, decode_content)

37
lib/requests_toolbelt/exceptions.py Normal file
View File

@@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
"""Collection of exceptions raised by requests-toolbelt."""
class StreamingError(Exception):
"""Used in :mod:`requests_toolbelt.downloadutils.stream`."""
pass
class VersionMismatchError(Exception):
"""Used to indicate a version mismatch in the version of requests required.
The feature in use requires a newer version of Requests to function
appropriately but the version installed is not sufficient.
"""
pass
class RequestsVersionTooOld(Warning):
"""Used to indicate that the Requests version is too old.
If the version of Requests is too old to support a feature, we will issue
this warning to the user.
"""
pass
class IgnoringGAECertificateValidation(Warning):
"""Used to indicate that given GAE validation behavior will be ignored.
If the user has tried to specify certificate validation when using the
insecure AppEngine adapter, it will be ignored (certificate validation will
remain off), so we will issue this warning to the user.
In :class:`requests_toolbelt.adapters.appengine.InsecureAppEngineAdapter`.
"""
pass

31
lib/requests_toolbelt/multipart/__init__.py Normal file
View File

@@ -0,0 +1,31 @@
"""
requests_toolbelt.multipart
===========================
See http://toolbelt.rtfd.org/ for documentation
:copyright: (c) 2014 by Ian Cordasco and Cory Benfield
:license: Apache v2.0, see LICENSE for more details
"""
from .encoder import MultipartEncoder, MultipartEncoderMonitor
from .decoder import MultipartDecoder
from .decoder import ImproperBodyPartContentException
from .decoder import NonMultipartContentTypeException
__title__ = 'requests-toolbelt'
__authors__ = 'Ian Cordasco, Cory Benfield'
__license__ = 'Apache v2.0'
__copyright__ = 'Copyright 2014 Ian Cordasco, Cory Benfield'
__all__ = [
'MultipartEncoder',
'MultipartEncoderMonitor',
'MultipartDecoder',
'ImproperBodyPartContentException',
'NonMultipartContentTypeException',
'__title__',
'__authors__',
'__license__',
'__copyright__',
]

156
lib/requests_toolbelt/multipart/decoder.py Normal file
View File

@@ -0,0 +1,156 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.multipart.decoder
===================================
This holds all the implementation details of the MultipartDecoder
"""
import sys
import email.parser
from .encoder import encode_with
from requests.structures import CaseInsensitiveDict
def _split_on_find(content, bound):
point = content.find(bound)
return content[:point], content[point + len(bound):]
class ImproperBodyPartContentException(Exception):
pass
class NonMultipartContentTypeException(Exception):
pass
def _header_parser(string, encoding):
major = sys.version_info[0]
if major == 3:
string = string.decode(encoding)
headers = email.parser.HeaderParser().parsestr(string).items()
return (
(encode_with(k, encoding), encode_with(v, encoding))
for k, v in headers
)
class BodyPart(object):
"""
The ``BodyPart`` object is a ``Response``-like interface to an individual
subpart of a multipart response. It is expected that these will
generally be created by objects of the ``MultipartDecoder`` class.
Like ``Response``, there is a ``CaseInsensitiveDict`` object named headers,
``content`` to access bytes, ``text`` to access unicode, and ``encoding``
to access the unicode codec.
"""
def __init__(self, content, encoding):
self.encoding = encoding
headers = {}
# Split into header section (if any) and the content
if b'\r\n\r\n' in content:
first, self.content = _split_on_find(content, b'\r\n\r\n')
if first != b'':
headers = _header_parser(first.lstrip(), encoding)
else:
raise ImproperBodyPartContentException(
'content does not contain CR-LF-CR-LF'
)
self.headers = CaseInsensitiveDict(headers)
@property
def text(self):
"""Content of the ``BodyPart`` in unicode."""
return self.content.decode(self.encoding)
class MultipartDecoder(object):
"""
The ``MultipartDecoder`` object parses the multipart payload of
a bytestring into a tuple of ``Response``-like ``BodyPart`` objects.
The basic usage is::
import requests
from requests_toolbelt import MultipartDecoder
response = request.get(url)
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
print(part.headers['content-type'])
If the multipart content is not from a response, basic usage is::
from requests_toolbelt import MultipartDecoder
decoder = MultipartDecoder(content, content_type)
for part in decoder.parts:
print(part.headers['content-type'])
For both these usages, there is an optional ``encoding`` parameter. This is
a string, which is the name of the unicode codec to use (default is
``'utf-8'``).
"""
def __init__(self, content, content_type, encoding='utf-8'):
#: Original Content-Type header
self.content_type = content_type
#: Response body encoding
self.encoding = encoding
#: Parsed parts of the multipart response body
self.parts = tuple()
self._find_boundary()
self._parse_body(content)
def _find_boundary(self):
ct_info = tuple(x.strip() for x in self.content_type.split(';'))
mimetype = ct_info[0]
if mimetype.split('/')[0].lower() != 'multipart':
raise NonMultipartContentTypeException(
"Unexpected mimetype in content-type: '{0}'".format(mimetype)
)
for item in ct_info[1:]:
attr, value = _split_on_find(
item,
'='
)
if attr.lower() == 'boundary':
self.boundary = encode_with(value.strip('"'), self.encoding)
@staticmethod
def _fix_first_part(part, boundary_marker):
bm_len = len(boundary_marker)
if boundary_marker == part[:bm_len]:
return part[bm_len:]
else:
return part
def _parse_body(self, content):
boundary = b''.join((b'--', self.boundary))
def body_part(part):
fixed = MultipartDecoder._fix_first_part(part, boundary)
return BodyPart(fixed, self.encoding)
def test_part(part):
return (part != b'' and
part != b'\r\n' and
part[:4] != b'--\r\n' and
part != b'--')
parts = content.split(b''.join((b'\r\n', boundary)))
self.parts = tuple(body_part(x) for x in parts if test_part(x))
@classmethod
def from_response(cls, response, encoding='utf-8'):
content = response.content
content_type = response.headers.get('content-type', None)
return cls(content, content_type, encoding)

655
lib/requests_toolbelt/multipart/encoder.py Normal file
View File

@@ -0,0 +1,655 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.multipart.encoder
===================================
This holds all of the implementation details of the MultipartEncoder
"""
import contextlib
import io
import os
from uuid import uuid4
import requests
from .._compat import fields
class FileNotSupportedError(Exception):
"""File not supported error."""
class MultipartEncoder(object):
"""
The ``MultipartEncoder`` object is a generic interface to the engine that
will create a ``multipart/form-data`` body for you.
The basic usage is:
.. code-block:: python
import requests
from requests_toolbelt import MultipartEncoder
encoder = MultipartEncoder({'field': 'value',
'other_field', 'other_value'})
r = requests.post('https://httpbin.org/post', data=encoder,
headers={'Content-Type': encoder.content_type})
If you do not need to take advantage of streaming the post body, you can
also do:
.. code-block:: python
r = requests.post('https://httpbin.org/post',
data=encoder.to_string(),
headers={'Content-Type': encoder.content_type})
If you want the encoder to use a specific order, you can use an
OrderedDict or more simply, a list of tuples:
.. code-block:: python
encoder = MultipartEncoder([('field', 'value'),
('other_field', 'other_value')])
.. versionchanged:: 0.4.0
You can also provide tuples as part values as you would provide them to
requests' ``files`` parameter.
.. code-block:: python
encoder = MultipartEncoder({
'field': ('file_name', b'{"a": "b"}', 'application/json',
{'X-My-Header': 'my-value'})
])
.. warning::
This object will end up directly in :mod:`httplib`. Currently,
:mod:`httplib` has a hard-coded read size of **8192 bytes**. This
means that it will loop until the file has been read and your upload
could take a while. This is **not** a bug in requests. A feature is
being considered for this object to allow you, the user, to specify
what size should be returned on a read. If you have opinions on this,
please weigh in on `this issue`_.
.. _this issue:
https://github.com/requests/toolbelt/issues/75
"""
def __init__(self, fields, boundary=None, encoding='utf-8'):
#: Boundary value either passed in by the user or created
self.boundary_value = boundary or uuid4().hex
# Computed boundary
self.boundary = '--{0}'.format(self.boundary_value)
#: Encoding of the data being passed in
self.encoding = encoding
# Pre-encoded boundary
self._encoded_boundary = b''.join([
encode_with(self.boundary, self.encoding),
encode_with('\r\n', self.encoding)
])
#: Fields provided by the user
self.fields = fields
#: Whether or not the encoder is finished
self.finished = False
#: Pre-computed parts of the upload
self.parts = []
# Pre-computed parts iterator
self._iter_parts = iter([])
# The part we're currently working with
self._current_part = None
# Cached computation of the body's length
self._len = None
# Our buffer
self._buffer = CustomBytesIO(encoding=encoding)
# Pre-compute each part's headers
self._prepare_parts()
# Load boundary into buffer
self._write_boundary()
@property
def len(self):
"""Length of the multipart/form-data body.
requests will first attempt to get the length of the body by calling
``len(body)`` and then by checking for the ``len`` attribute.
On 32-bit systems, the ``__len__`` method cannot return anything
larger than an integer (in C) can hold. If the total size of the body
is even slightly larger than 4GB users will see an OverflowError. This
manifested itself in `bug #80`_.
As such, we now calculate the length lazily as a property.
.. _bug #80:
https://github.com/requests/toolbelt/issues/80
"""
# If _len isn't already calculated, calculate, return, and set it
return self._len or self._calculate_length()
def __repr__(self):
return '<MultipartEncoder: {0!r}>'.format(self.fields)
def _calculate_length(self):
"""
This uses the parts to calculate the length of the body.
This returns the calculated length so __len__ can be lazy.
"""
boundary_len = len(self.boundary) # Length of --{boundary}
# boundary length + header length + body length + len('\r\n') * 2
self._len = sum(
(boundary_len + total_len(p) + 4) for p in self.parts
) + boundary_len + 4
return self._len
def _calculate_load_amount(self, read_size):
"""This calculates how many bytes need to be added to the buffer.
When a consumer read's ``x`` from the buffer, there are two cases to
satisfy:
1. Enough data in the buffer to return the requested amount
2. Not enough data
This function uses the amount of unread bytes in the buffer and
determines how much the Encoder has to load before it can return the
requested amount of bytes.
:param int read_size: the number of bytes the consumer requests
:returns: int -- the number of bytes that must be loaded into the
buffer before the read can be satisfied. This will be strictly
non-negative
"""
amount = read_size - total_len(self._buffer)
return amount if amount > 0 else 0
def _load(self, amount):
"""Load ``amount`` number of bytes into the buffer."""
self._buffer.smart_truncate()
part = self._current_part or self._next_part()
while amount == -1 or amount > 0:
written = 0
if part and not part.bytes_left_to_write():
written += self._write(b'\r\n')
written += self._write_boundary()
part = self._next_part()
if not part:
written += self._write_closing_boundary()
self.finished = True
break
written += part.write_to(self._buffer, amount)
if amount != -1:
amount -= written
def _next_part(self):
try:
p = self._current_part = next(self._iter_parts)
except StopIteration:
p = None
return p
def _iter_fields(self):
_fields = self.fields
if hasattr(self.fields, 'items'):
_fields = list(self.fields.items())
for k, v in _fields:
file_name = None
file_type = None
file_headers = None
if isinstance(v, (list, tuple)):
if len(v) == 2:
file_name, file_pointer = v
elif len(v) == 3:
file_name, file_pointer, file_type = v
else:
file_name, file_pointer, file_type, file_headers = v
else:
file_pointer = v
field = fields.RequestField(name=k, data=file_pointer,
filename=file_name,
headers=file_headers)
field.make_multipart(content_type=file_type)
yield field
def _prepare_parts(self):
"""This uses the fields provided by the user and creates Part objects.
It populates the `parts` attribute and uses that to create a
generator for iteration.
"""
enc = self.encoding
self.parts = [Part.from_field(f, enc) for f in self._iter_fields()]
self._iter_parts = iter(self.parts)
def _write(self, bytes_to_write):
"""Write the bytes to the end of the buffer.
:param bytes bytes_to_write: byte-string (or bytearray) to append to
the buffer
:returns: int -- the number of bytes written
"""
return self._buffer.append(bytes_to_write)
def _write_boundary(self):
"""Write the boundary to the end of the buffer."""
return self._write(self._encoded_boundary)
def _write_closing_boundary(self):
"""Write the bytes necessary to finish a multipart/form-data body."""
with reset(self._buffer):
self._buffer.seek(-2, 2)
self._buffer.write(b'--\r\n')
return 2
def _write_headers(self, headers):
"""Write the current part's headers to the buffer."""
return self._write(encode_with(headers, self.encoding))
@property
def content_type(self):
return str(
'multipart/form-data; boundary={0}'.format(self.boundary_value)
)
def to_string(self):
"""Return the entirety of the data in the encoder.
.. note::
This simply reads all of the data it can. If you have started
streaming or reading data from the encoder, this method will only
return whatever data is left in the encoder.
.. note::
This method affects the internal state of the encoder. Calling
this method will exhaust the encoder.
:returns: the multipart message
:rtype: bytes
"""
return self.read()
def read(self, size=-1):
"""Read data from the streaming encoder.
:param int size: (optional), If provided, ``read`` will return exactly
that many bytes. If it is not provided, it will return the
remaining bytes.
:returns: bytes
"""
if self.finished:
return self._buffer.read(size)
bytes_to_load = size
if bytes_to_load != -1 and bytes_to_load is not None:
bytes_to_load = self._calculate_load_amount(int(size))
self._load(bytes_to_load)
return self._buffer.read(size)
def IDENTITY(monitor):
return monitor
class MultipartEncoderMonitor(object):
"""
An object used to monitor the progress of a :class:`MultipartEncoder`.
The :class:`MultipartEncoder` should only be responsible for preparing and
streaming the data. For anyone who wishes to monitor it, they shouldn't be
using that instance to manage that as well. Using this class, they can
monitor an encoder and register a callback. The callback receives the
instance of the monitor.
To use this monitor, you construct your :class:`MultipartEncoder` as you
normally would.
.. code-block:: python
from requests_toolbelt import (MultipartEncoder,
MultipartEncoderMonitor)
import requests
def callback(monitor):
# Do something with this information
pass
m = MultipartEncoder(fields={'field0': 'value0'})
monitor = MultipartEncoderMonitor(m, callback)
headers = {'Content-Type': monitor.content_type}
r = requests.post('https://httpbin.org/post', data=monitor,
headers=headers)
Alternatively, if your use case is very simple, you can use the following
pattern.
.. code-block:: python
from requests_toolbelt import MultipartEncoderMonitor
import requests
def callback(monitor):
# Do something with this information
pass
monitor = MultipartEncoderMonitor.from_fields(
fields={'field0': 'value0'}, callback
)
headers = {'Content-Type': montior.content_type}
r = requests.post('https://httpbin.org/post', data=monitor,
headers=headers)
"""
def __init__(self, encoder, callback=None):
#: Instance of the :class:`MultipartEncoder` being monitored
self.encoder = encoder
#: Optionally function to call after a read
self.callback = callback or IDENTITY
#: Number of bytes already read from the :class:`MultipartEncoder`
#: instance
self.bytes_read = 0
#: Avoid the same problem in bug #80
self.len = self.encoder.len
@classmethod
def from_fields(cls, fields, boundary=None, encoding='utf-8',
callback=None):
encoder = MultipartEncoder(fields, boundary, encoding)
return cls(encoder, callback)
@property
def content_type(self):
return self.encoder.content_type
def to_string(self):
return self.read()
def read(self, size=-1):
string = self.encoder.read(size)
self.bytes_read += len(string)
self.callback(self)
return string
def encode_with(string, encoding):
"""Encoding ``string`` with ``encoding`` if necessary.
:param str string: If string is a bytes object, it will not encode it.
Otherwise, this function will encode it with the provided encoding.
:param str encoding: The encoding with which to encode string.
:returns: encoded bytes object
"""
if not (string is None or isinstance(string, bytes)):
return string.encode(encoding)
return string
def readable_data(data, encoding):
"""Coerce the data to an object with a ``read`` method."""
if hasattr(data, 'read'):
return data
return CustomBytesIO(data, encoding)
def total_len(o):
if hasattr(o, '__len__'):
return len(o)
if hasattr(o, 'len'):
return o.len
if hasattr(o, 'fileno'):
try:
fileno = o.fileno()
except io.UnsupportedOperation:
pass
else:
return os.fstat(fileno).st_size
if hasattr(o, 'getvalue'):
# e.g. BytesIO, cStringIO.StringIO
return len(o.getvalue())
@contextlib.contextmanager
def reset(buffer):
"""Keep track of the buffer's current position and write to the end.
This is a context manager meant to be used when adding data to the buffer.
It eliminates the need for every function to be concerned with the
position of the cursor in the buffer.
"""
original_position = buffer.tell()
buffer.seek(0, 2)
yield
buffer.seek(original_position, 0)
def coerce_data(data, encoding):
"""Ensure that every object's __len__ behaves uniformly."""
if not isinstance(data, CustomBytesIO):
if hasattr(data, 'getvalue'):
return CustomBytesIO(data.getvalue(), encoding)
if hasattr(data, 'fileno'):
return FileWrapper(data)
if not hasattr(data, 'read'):
return CustomBytesIO(data, encoding)
return data
def to_list(fields):
if hasattr(fields, 'items'):
return list(fields.items())
return list(fields)
class Part(object):
def __init__(self, headers, body):
self.headers = headers
self.body = body
self.headers_unread = True
self.len = len(self.headers) + total_len(self.body)
@classmethod
def from_field(cls, field, encoding):
"""Create a part from a Request Field generated by urllib3."""
headers = encode_with(field.render_headers(), encoding)
body = coerce_data(field.data, encoding)
return cls(headers, body)
def bytes_left_to_write(self):
"""Determine if there are bytes left to write.
:returns: bool -- ``True`` if there are bytes left to write, otherwise
``False``
"""
to_read = 0
if self.headers_unread:
to_read += len(self.headers)
return (to_read + total_len(self.body)) > 0
def write_to(self, buffer, size):
"""Write the requested amount of bytes to the buffer provided.
The number of bytes written may exceed size on the first read since we
load the headers ambitiously.
:param CustomBytesIO buffer: buffer we want to write bytes to
:param int size: number of bytes requested to be written to the buffer
:returns: int -- number of bytes actually written
"""
written = 0
if self.headers_unread:
written += buffer.append(self.headers)
self.headers_unread = False
while total_len(self.body) > 0 and (size == -1 or written < size):
amount_to_read = size
if size != -1:
amount_to_read = size - written
written += buffer.append(self.body.read(amount_to_read))
return written
class CustomBytesIO(io.BytesIO):
def __init__(self, buffer=None, encoding='utf-8'):
buffer = encode_with(buffer, encoding)
super(CustomBytesIO, self).__init__(buffer)
def _get_end(self):
current_pos = self.tell()
self.seek(0, 2)
length = self.tell()
self.seek(current_pos, 0)
return length
@property
def len(self):
length = self._get_end()
return length - self.tell()
def append(self, bytes):
with reset(self):
written = self.write(bytes)
return written
def smart_truncate(self):
to_be_read = total_len(self)
already_read = self._get_end() - to_be_read
if already_read >= to_be_read:
old_bytes = self.read()
self.seek(0, 0)
self.truncate()
self.write(old_bytes)
self.seek(0, 0) # We want to be at the beginning
class FileWrapper(object):
def __init__(self, file_object):
self.fd = file_object
@property
def len(self):
return total_len(self.fd) - self.fd.tell()
def read(self, length=-1):
return self.fd.read(length)
class FileFromURLWrapper(object):
"""File from URL wrapper.
The :class:`FileFromURLWrapper` object gives you the ability to stream file
from provided URL in chunks by :class:`MultipartEncoder`.
Provide a stateless solution for streaming file from one server to another.
You can use the :class:`FileFromURLWrapper` without a session or with
a session as demonstated by the examples below:
.. code-block:: python
# no session
import requests
from requests_toolbelt import MultipartEncoder, FileFromURLWrapper
url = 'https://httpbin.org/image/png'
streaming_encoder = MultipartEncoder(
fields={
'file': FileFromURLWrapper(url)
}
)
r = requests.post(
'https://httpbin.org/post', data=streaming_encoder,
headers={'Content-Type': streaming_encoder.content_type}
)
.. code-block:: python
# using a session
import requests
from requests_toolbelt import MultipartEncoder, FileFromURLWrapper
session = requests.Session()
url = 'https://httpbin.org/image/png'
streaming_encoder = MultipartEncoder(
fields={
'file': FileFromURLWrapper(url, session=session)
}
)
r = session.post(
'https://httpbin.org/post', data=streaming_encoder,
headers={'Content-Type': streaming_encoder.content_type}
)
"""
def __init__(self, file_url, session=None):
self.session = session or requests.Session()
requested_file = self._request_for_file(file_url)
self.len = int(requested_file.headers['content-length'])
self.raw_data = requested_file.raw
def _request_for_file(self, file_url):
"""Make call for file under provided URL."""
response = self.session.get(file_url, stream=True)
content_length = response.headers.get('content-length', None)
if content_length is None:
error_msg = (
"Data from provided URL {url} is not supported. Lack of "
"content-length Header in requested file response.".format(
url=file_url)
)
raise FileNotSupportedError(error_msg)
elif not content_length.isdigit():
error_msg = (
"Data from provided URL {url} is not supported. content-length"
" header value is not a digit.".format(url=file_url)
)
raise FileNotSupportedError(error_msg)
return response
def read(self, chunk_size):
"""Read file in chunks."""
chunk_size = chunk_size if chunk_size >= 0 else self.len
chunk = self.raw_data.read(chunk_size) or b''
self.len -= len(chunk) if chunk else 0 # left to read
return chunk

70
lib/requests_toolbelt/sessions.py Normal file
View File

@@ -0,0 +1,70 @@
import requests
from ._compat import urljoin
class BaseUrlSession(requests.Session):
"""A Session with a URL that all requests will use as a base.
Let's start by looking at an example:
.. code-block:: python
>>> from requests_toolbelt import sessions
>>> s = sessions.BaseUrlSession(
... base_url='https://example.com/resource/')
>>> r = s.get('sub-resource/' params={'foo': 'bar'})
>>> print(r.request.url)
https://example.com/resource/sub-resource/?foo=bar
Our call to the ``get`` method will make a request to the URL passed in
when we created the Session and the partial resource name we provide.
We implement this by overriding the ``request`` method so most uses of a
Session are covered. (This, however, precludes the use of PreparedRequest
objects).
.. note::
The base URL that you provide and the path you provide are **very**
important.
Let's look at another *similar* example
.. code-block:: python
>>> from requests_toolbelt import sessions
>>> s = sessions.BaseUrlSession(
... base_url='https://example.com/resource/')
>>> r = s.get('/sub-resource/' params={'foo': 'bar'})
>>> print(r.request.url)
https://example.com/sub-resource/?foo=bar
The key difference here is that we called ``get`` with ``/sub-resource/``,
i.e., there was a leading ``/``. This changes how we create the URL
because we rely on :mod:`urllib.parse.urljoin`.
To override how we generate the URL, sub-class this method and override the
``create_url`` method.
Based on implementation from
https://github.com/kennethreitz/requests/issues/2554#issuecomment-109341010
"""
base_url = None
def __init__(self, base_url=None):
if base_url:
self.base_url = base_url
super(BaseUrlSession, self).__init__()
def request(self, method, url, *args, **kwargs):
"""Send the request after generating the complete URL."""
url = self.create_url(url)
return super(BaseUrlSession, self).request(
method, url, *args, **kwargs
)
def create_url(self, url):
"""Create the URL based off this partial path."""
return urljoin(self.base_url, url)

116
lib/requests_toolbelt/streaming_iterator.py Normal file
View File

@@ -0,0 +1,116 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.streaming_iterator
====================================
This holds the implementation details for the :class:`StreamingIterator`. It
is designed for the case where you, the user, know the size of the upload but
need to provide the data as an iterator. This class will allow you to specify
the size and stream the data without using a chunked transfer-encoding.
"""
from requests.utils import super_len
from .multipart.encoder import CustomBytesIO, encode_with
class StreamingIterator(object):
"""
This class provides a way of allowing iterators with a known size to be
streamed instead of chunked.
In requests, if you pass in an iterator it assumes you want to use
chunked transfer-encoding to upload the data, which not all servers
support well. Additionally, you may want to set the content-length
yourself to avoid this but that will not work. The only way to preempt
requests using a chunked transfer-encoding and forcing it to stream the
uploads is to mimic a very specific interace. Instead of having to know
these details you can instead just use this class. You simply provide the
size and iterator and pass the instance of StreamingIterator to requests
via the data parameter like so:
.. code-block:: python
from requests_toolbelt import StreamingIterator
import requests
# Let iterator be some generator that you already have and size be
# the size of the data produced by the iterator
r = requests.post(url, data=StreamingIterator(size, iterator))
You can also pass file-like objects to :py:class:`StreamingIterator` in
case requests can't determize the filesize itself. This is the case with
streaming file objects like ``stdin`` or any sockets. Wrapping e.g. files
that are on disk with ``StreamingIterator`` is unnecessary, because
requests can determine the filesize itself.
Naturally, you should also set the `Content-Type` of your upload
appropriately because the toolbelt will not attempt to guess that for you.
"""
def __init__(self, size, iterator, encoding='utf-8'):
#: The expected size of the upload
self.size = int(size)
if self.size < 0:
raise ValueError(
'The size of the upload must be a positive integer'
)
#: Attribute that requests will check to determine the length of the
#: body. See bug #80 for more details
self.len = self.size
#: Encoding the input data is using
self.encoding = encoding
#: The iterator used to generate the upload data
self.iterator = iterator
if hasattr(iterator, 'read'):
self._file = iterator
else:
self._file = _IteratorAsBinaryFile(iterator, encoding)
def read(self, size=-1):
return encode_with(self._file.read(size), self.encoding)
class _IteratorAsBinaryFile(object):
def __init__(self, iterator, encoding='utf-8'):
#: The iterator used to generate the upload data
self.iterator = iterator
#: Encoding the iterator is using
self.encoding = encoding
# The buffer we use to provide the correct number of bytes requested
# during a read
self._buffer = CustomBytesIO()
def _get_bytes(self):
try:
return encode_with(next(self.iterator), self.encoding)
except StopIteration:
return b''
def _load_bytes(self, size):
self._buffer.smart_truncate()
amount_to_load = size - super_len(self._buffer)
bytes_to_append = True
while amount_to_load > 0 and bytes_to_append:
bytes_to_append = self._get_bytes()
amount_to_load -= self._buffer.append(bytes_to_append)
def read(self, size=-1):
size = int(size)
if size == -1:
return b''.join(self.iterator)
self._load_bytes(size)
return self._buffer.read(size)

97
lib/requests_toolbelt/threaded/__init__.py Normal file
View File

@@ -0,0 +1,97 @@
"""
This module provides the API for ``requests_toolbelt.threaded``.
The module provides a clean and simple API for making requests via a thread
pool. The thread pool will use sessions for increased performance.
A simple use-case is:
.. code-block:: python
from requests_toolbelt import threaded
urls_to_get = [{
'url': 'https://api.github.com/users/sigmavirus24',
'method': 'GET',
}, {
'url': 'https://api.github.com/repos/requests/toolbelt',
'method': 'GET',
}, {
'url': 'https://google.com',
'method': 'GET',
}]
responses, errors = threaded.map(urls_to_get)
By default, the threaded submodule will detect the number of CPUs your
computer has and use that if no other number of processes is selected. To
change this, always use the keyword argument ``num_processes``. Using the
above example, we would expand it like so:
.. code-block:: python
responses, errors = threaded.map(urls_to_get, num_processes=10)
You can also customize how a :class:`requests.Session` is initialized by
creating a callback function:
.. code-block:: python
from requests_toolbelt import user_agent
def initialize_session(session):
session.headers['User-Agent'] = user_agent('my-scraper', '0.1')
session.headers['Accept'] = 'application/json'
responses, errors = threaded.map(urls_to_get,
initializer=initialize_session)
.. autofunction:: requests_toolbelt.threaded.map
Inspiration is blatantly drawn from the standard library's multiprocessing
library. See the following references:
- multiprocessing's `pool source`_
- map and map_async `inspiration`_
.. _pool source:
https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py
.. _inspiration:
https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py#l340
"""
from . import pool
from .._compat import queue
def map(requests, **kwargs):
r"""Simple interface to the threaded Pool object.
This function takes a list of dictionaries representing requests to make
using Sessions in threads and returns a tuple where the first item is
a generator of successful responses and the second is a generator of
exceptions.
:param list requests:
Collection of dictionaries representing requests to make with the Pool
object.
:param \*\*kwargs:
Keyword arguments that are passed to the
:class:`~requests_toolbelt.threaded.pool.Pool` object.
:returns: Tuple of responses and exceptions from the pool
:rtype: (:class:`~requests_toolbelt.threaded.pool.ThreadResponse`,
:class:`~requests_toolbelt.threaded.pool.ThreadException`)
"""
if not (requests and all(isinstance(r, dict) for r in requests)):
raise ValueError('map expects a list of dictionaries.')
# Build our queue of requests
job_queue = queue.Queue()
for request in requests:
job_queue.put(request)
# Ensure the user doesn't try to pass their own job_queue
kwargs['job_queue'] = job_queue
threadpool = pool.Pool(**kwargs)
threadpool.join_all()
return threadpool.responses(), threadpool.exceptions()

211
lib/requests_toolbelt/threaded/pool.py Normal file
View File

@@ -0,0 +1,211 @@
"""Module implementing the Pool for :mod:``requests_toolbelt.threaded``."""
import multiprocessing
import requests
from . import thread
from .._compat import queue
class Pool(object):
"""Pool that manages the threads containing sessions.
:param queue:
The queue you're expected to use to which you should add items.
:type queue: queue.Queue
:param initializer:
Function used to initialize an instance of ``session``.
:type initializer: collections.Callable
:param auth_generator:
Function used to generate new auth credentials for the session.
:type auth_generator: collections.Callable
:param int num_process:
Number of threads to create.
:param session:
:type session: requests.Session
"""
def __init__(self, job_queue, initializer=None, auth_generator=None,
num_processes=None, session=requests.Session):
if num_processes is None:
num_processes = multiprocessing.cpu_count() or 1
if num_processes < 1:
raise ValueError("Number of processes should at least be 1.")
self._job_queue = job_queue
self._response_queue = queue.Queue()
self._exc_queue = queue.Queue()
self._processes = num_processes
self._initializer = initializer or _identity
self._auth = auth_generator or _identity
self._session = session
self._pool = [
thread.SessionThread(self._new_session(), self._job_queue,
self._response_queue, self._exc_queue)
for _ in range(self._processes)
]
def _new_session(self):
return self._auth(self._initializer(self._session()))
@classmethod
def from_exceptions(cls, exceptions, **kwargs):
r"""Create a :class:`~Pool` from an :class:`~ThreadException`\ s.
Provided an iterable that provides :class:`~ThreadException` objects,
this classmethod will generate a new pool to retry the requests that
caused the exceptions.
:param exceptions:
Iterable that returns :class:`~ThreadException`
:type exceptions: iterable
:param kwargs:
Keyword arguments passed to the :class:`~Pool` initializer.
:returns: An initialized :class:`~Pool` object.
:rtype: :class:`~Pool`
"""
job_queue = queue.Queue()
for exc in exceptions:
job_queue.put(exc.request_kwargs)
return cls(job_queue=job_queue, **kwargs)
@classmethod
def from_urls(cls, urls, request_kwargs=None, **kwargs):
"""Create a :class:`~Pool` from an iterable of URLs.
:param urls:
Iterable that returns URLs with which we create a pool.
:type urls: iterable
:param dict request_kwargs:
Dictionary of other keyword arguments to provide to the request
method.
:param kwargs:
Keyword arguments passed to the :class:`~Pool` initializer.
:returns: An initialized :class:`~Pool` object.
:rtype: :class:`~Pool`
"""
request_dict = {'method': 'GET'}
request_dict.update(request_kwargs or {})
job_queue = queue.Queue()
for url in urls:
job = request_dict.copy()
job.update({'url': url})
job_queue.put(job)
return cls(job_queue=job_queue, **kwargs)
def exceptions(self):
"""Iterate over all the exceptions in the pool.
:returns: Generator of :class:`~ThreadException`
"""
while True:
exc = self.get_exception()
if exc is None:
break
yield exc
def get_exception(self):
"""Get an exception from the pool.
:rtype: :class:`~ThreadException`
"""
try:
(request, exc) = self._exc_queue.get_nowait()
except queue.Empty:
return None
else:
return ThreadException(request, exc)
def get_response(self):
"""Get a response from the pool.
:rtype: :class:`~ThreadResponse`
"""
try:
(request, response) = self._response_queue.get_nowait()
except queue.Empty:
return None
else:
return ThreadResponse(request, response)
def responses(self):
"""Iterate over all the responses in the pool.
:returns: Generator of :class:`~ThreadResponse`
"""
while True:
resp = self.get_response()
if resp is None:
break
yield resp
def join_all(self):
"""Join all the threads to the master thread."""
for session_thread in self._pool:
session_thread.join()
class ThreadProxy(object):
proxied_attr = None
def __getattr__(self, attr):
"""Proxy attribute accesses to the proxied object."""
get = object.__getattribute__
if attr not in self.attrs:
response = get(self, self.proxied_attr)
return getattr(response, attr)
else:
return get(self, attr)
class ThreadResponse(ThreadProxy):
"""A wrapper around a requests Response object.
This will proxy most attribute access actions to the Response object. For
example, if you wanted the parsed JSON from the response, you might do:
.. code-block:: python
thread_response = pool.get_response()
json = thread_response.json()
"""
proxied_attr = 'response'
attrs = frozenset(['request_kwargs', 'response'])
def __init__(self, request_kwargs, response):
#: The original keyword arguments provided to the queue
self.request_kwargs = request_kwargs
#: The wrapped response
self.response = response
class ThreadException(ThreadProxy):
"""A wrapper around an exception raised during a request.
This will proxy most attribute access actions to the exception object. For
example, if you wanted the message from the exception, you might do:
.. code-block:: python
thread_exc = pool.get_exception()
msg = thread_exc.message
"""
proxied_attr = 'exception'
attrs = frozenset(['request_kwargs', 'exception'])
def __init__(self, request_kwargs, exception):
#: The original keyword arguments provided to the queue
self.request_kwargs = request_kwargs
#: The captured and wrapped exception
self.exception = exception
def _identity(session_obj):
return session_obj
__all__ = ['ThreadException', 'ThreadResponse', 'Pool']

53
lib/requests_toolbelt/threaded/thread.py Normal file
View File

@@ -0,0 +1,53 @@
"""Module containing the SessionThread class."""
import threading
import uuid
import requests.exceptions as exc
from .._compat import queue
class SessionThread(object):
def __init__(self, initialized_session, job_queue, response_queue,
exception_queue):
self._session = initialized_session
self._jobs = job_queue
self._create_worker()
self._responses = response_queue
self._exceptions = exception_queue
def _create_worker(self):
self._worker = threading.Thread(
target=self._make_request,
name=uuid.uuid4(),
)
self._worker.daemon = True
self._worker._state = 0
self._worker.start()
def _handle_request(self, kwargs):
try:
response = self._session.request(**kwargs)
except exc.RequestException as e:
self._exceptions.put((kwargs, e))
else:
self._responses.put((kwargs, response))
finally:
self._jobs.task_done()
def _make_request(self):
while True:
try:
kwargs = self._jobs.get_nowait()
except queue.Empty:
break
self._handle_request(kwargs)
def is_alive(self):
"""Proxy to the thread's ``is_alive`` method."""
return self._worker.is_alive()
def join(self):
"""Join this thread to the master thread."""
self._worker.join()

0
lib/requests_toolbelt/utils/__init__.py Normal file
View File

91
lib/requests_toolbelt/utils/deprecated.py Normal file
View File

@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""A collection of functions deprecated in requests.utils."""
import re
import sys
from requests import utils
find_charset = re.compile(
br'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I
).findall
find_pragma = re.compile(
br'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I
).findall
find_xml = re.compile(
br'^<\?xml.*?encoding=["\']*(.+?)["\'>]'
).findall
def get_encodings_from_content(content):
"""Return encodings from given content string.
.. code-block:: python
import requests
from requests_toolbelt.utils import deprecated
r = requests.get(url)
encodings = deprecated.get_encodings_from_content(r)
:param content: bytestring to extract encodings from
:type content: bytes
:return: encodings detected in the provided content
:rtype: list(str)
"""
encodings = (find_charset(content) + find_pragma(content)
+ find_xml(content))
if (3, 0) <= sys.version_info < (4, 0):
encodings = [encoding.decode('utf8') for encoding in encodings]
return encodings
def get_unicode_from_response(response):
"""Return the requested content back in unicode.
This will first attempt to retrieve the encoding from the response
headers. If that fails, it will use
:func:`requests_toolbelt.utils.deprecated.get_encodings_from_content`
to determine encodings from HTML elements.
.. code-block:: python
import requests
from requests_toolbelt.utils import deprecated
r = requests.get(url)
text = deprecated.get_unicode_from_response(r)
:param response: Response object to get unicode content from.
:type response: requests.models.Response
"""
tried_encodings = set()
# Try charset from content-type
encoding = utils.get_encoding_from_headers(response.headers)
if encoding:
try:
return str(response.content, encoding)
except UnicodeError:
tried_encodings.add(encoding.lower())
encodings = get_encodings_from_content(response.content)
for _encoding in encodings:
_encoding = _encoding.lower()
if _encoding in tried_encodings:
continue
try:
return str(response.content, _encoding)
except UnicodeError:
tried_encodings.add(_encoding)
# Fall back:
if encoding:
try:
return str(response.content, encoding, errors='replace')
except TypeError:
pass
return response.text

197
lib/requests_toolbelt/utils/dump.py Normal file
View File

@@ -0,0 +1,197 @@
"""This module provides functions for dumping information about responses."""
import collections
from requests import compat
__all__ = ('dump_response', 'dump_all')
HTTP_VERSIONS = {
9: b'0.9',
10: b'1.0',
11: b'1.1',
}
_PrefixSettings = collections.namedtuple('PrefixSettings',
['request', 'response'])
class PrefixSettings(_PrefixSettings):
def __new__(cls, request, response):
request = _coerce_to_bytes(request)
response = _coerce_to_bytes(response)
return super(PrefixSettings, cls).__new__(cls, request, response)
def _get_proxy_information(response):
if getattr(response.connection, 'proxy_manager', False):
proxy_info = {}
request_url = response.request.url
if request_url.startswith('https://'):
proxy_info['method'] = 'CONNECT'
proxy_info['request_path'] = request_url
return proxy_info
return None
def _format_header(name, value):
return (_coerce_to_bytes(name) + b': ' + _coerce_to_bytes(value) +
b'\r\n')
def _build_request_path(url, proxy_info):
uri = compat.urlparse(url)
proxy_url = proxy_info.get('request_path')
if proxy_url is not None:
request_path = _coerce_to_bytes(proxy_url)
return request_path, uri
request_path = _coerce_to_bytes(uri.path)
if uri.query:
request_path += b'?' + _coerce_to_bytes(uri.query)
return request_path, uri
def _dump_request_data(request, prefixes, bytearr, proxy_info=None):
if proxy_info is None:
proxy_info = {}
prefix = prefixes.request
method = _coerce_to_bytes(proxy_info.pop('method', request.method))
request_path, uri = _build_request_path(request.url, proxy_info)
# <prefix><METHOD> <request-path> HTTP/1.1
bytearr.extend(prefix + method + b' ' + request_path + b' HTTP/1.1\r\n')
# <prefix>Host: <request-host> OR host header specified by user
headers = request.headers.copy()
host_header = _coerce_to_bytes(headers.pop('Host', uri.netloc))
bytearr.extend(prefix + b'Host: ' + host_header + b'\r\n')
for name, value in headers.items():
bytearr.extend(prefix + _format_header(name, value))
bytearr.extend(prefix + b'\r\n')
if request.body:
if isinstance(request.body, compat.basestring):
bytearr.extend(prefix + _coerce_to_bytes(request.body))
else:
# In the event that the body is a file-like object, let's not try
# to read everything into memory.
bytearr.extend(b'<< Request body is not a string-like type >>')
bytearr.extend(b'\r\n')
def _dump_response_data(response, prefixes, bytearr):
prefix = prefixes.response
# Let's interact almost entirely with urllib3's response
raw = response.raw
# Let's convert the version int from httplib to bytes
version_str = HTTP_VERSIONS.get(raw.version, b'?')
# <prefix>HTTP/<version_str> <status_code> <reason>
bytearr.extend(prefix + b'HTTP/' + version_str + b' ' +
str(raw.status).encode('ascii') + b' ' +
_coerce_to_bytes(response.reason) + b'\r\n')
headers = raw.headers
for name in headers.keys():
for value in headers.getlist(name):
bytearr.extend(prefix + _format_header(name, value))
bytearr.extend(prefix + b'\r\n')
bytearr.extend(response.content)
def _coerce_to_bytes(data):
if not isinstance(data, bytes) and hasattr(data, 'encode'):
data = data.encode('utf-8')
# Don't bail out with an exception if data is None
return data if data is not None else b''
def dump_response(response, request_prefix=b'< ', response_prefix=b'> ',
data_array=None):
"""Dump a single request-response cycle's information.
This will take a response object and dump only the data that requests can
see for that single request-response cycle.
Example::
import requests
from requests_toolbelt.utils import dump
resp = requests.get('https://api.github.com/users/sigmavirus24')
data = dump.dump_response(resp)
print(data.decode('utf-8'))
:param response:
The response to format
:type response: :class:`requests.Response`
:param request_prefix: (*optional*)
Bytes to prefix each line of the request data
:type request_prefix: :class:`bytes`
:param response_prefix: (*optional*)
Bytes to prefix each line of the response data
:type response_prefix: :class:`bytes`
:param data_array: (*optional*)
Bytearray to which we append the request-response cycle data
:type data_array: :class:`bytearray`
:returns: Formatted bytes of request and response information.
:rtype: :class:`bytearray`
"""
data = data_array if data_array is not None else bytearray()
prefixes = PrefixSettings(request_prefix, response_prefix)
if not hasattr(response, 'request'):
raise ValueError('Response has no associated request')
proxy_info = _get_proxy_information(response)
_dump_request_data(response.request, prefixes, data,
proxy_info=proxy_info)
_dump_response_data(response, prefixes, data)
return data
def dump_all(response, request_prefix=b'< ', response_prefix=b'> '):
"""Dump all requests and responses including redirects.
This takes the response returned by requests and will dump all
request-response pairs in the redirect history in order followed by the
final request-response.
Example::
import requests
from requests_toolbelt.utils import dump
resp = requests.get('https://httpbin.org/redirect/5')
data = dump.dump_all(resp)
print(data.decode('utf-8'))
:param response:
The response to format
:type response: :class:`requests.Response`
:param request_prefix: (*optional*)
Bytes to prefix each line of the request data
:type request_prefix: :class:`bytes`
:param response_prefix: (*optional*)
Bytes to prefix each line of the response data
:type response_prefix: :class:`bytes`
:returns: Formatted bytes of request and response information.
:rtype: :class:`bytearray`
"""
data = bytearray()
history = list(response.history[:])
history.append(response)
for response in history:
dump_response(response, request_prefix, response_prefix, data)
return data

108
lib/requests_toolbelt/utils/formdata.py Normal file
View File

@@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
"""Implementation of nested form-data encoding function(s)."""
from .._compat import basestring
from .._compat import urlencode as _urlencode
__all__ = ('urlencode',)
def urlencode(query, *args, **kwargs):
"""Handle nested form-data queries and serialize them appropriately.
There are times when a website expects a nested form data query to be sent
but, the standard library's urlencode function does not appropriately
handle the nested structures. In that case, you need this function which
will flatten the structure first and then properly encode it for you.
When using this to send data in the body of a request, make sure you
specify the appropriate Content-Type header for the request.
.. code-block:: python
import requests
from requests_toolbelt.utils import formdata
query = {
'my_dict': {
'foo': 'bar',
'biz': 'baz",
},
'a': 'b',
}
resp = requests.get(url, params=formdata.urlencode(query))
# or
resp = requests.post(
url,
data=formdata.urlencode(query),
headers={
'Content-Type': 'application/x-www-form-urlencoded'
},
)
Similarly, you can specify a list of nested tuples, e.g.,
.. code-block:: python
import requests
from requests_toolbelt.utils import formdata
query = [
('my_list', [
('foo', 'bar'),
('biz', 'baz'),
]),
('a', 'b'),
]
resp = requests.get(url, params=formdata.urlencode(query))
# or
resp = requests.post(
url,
data=formdata.urlencode(query),
headers={
'Content-Type': 'application/x-www-form-urlencoded'
},
)
For additional parameter and return information, see the official
`urlencode`_ documentation.
.. _urlencode:
https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode
"""
expand_classes = (dict, list, tuple)
original_query_list = _to_kv_list(query)
if not all(_is_two_tuple(i) for i in original_query_list):
raise ValueError("Expected query to be able to be converted to a "
"list comprised of length 2 tuples.")
query_list = original_query_list
while any(isinstance(v, expand_classes) for _, v in query_list):
query_list = _expand_query_values(query_list)
return _urlencode(query_list, *args, **kwargs)
def _to_kv_list(dict_or_list):
if hasattr(dict_or_list, 'items'):
return list(dict_or_list.items())
return dict_or_list
def _is_two_tuple(item):
return isinstance(item, (list, tuple)) and len(item) == 2
def _expand_query_values(original_query_list):
query_list = []
for key, value in original_query_list:
if isinstance(value, basestring):
query_list.append((key, value))
else:
key_fmt = key + '[%s]'
value_list = _to_kv_list(value)
query_list.extend((key_fmt % k, v) for k, v in value_list)
return query_list

143
lib/requests_toolbelt/utils/user_agent.py Normal file
View File

@@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
import collections
import platform
import sys
def user_agent(name, version, extras=None):
"""Return an internet-friendly user_agent string.
The majority of this code has been wilfully stolen from the equivalent
function in Requests.
:param name: The intended name of the user-agent, e.g. "python-requests".
:param version: The version of the user-agent, e.g. "0.0.1".
:param extras: List of two-item tuples that are added to the user-agent
string.
:returns: Formatted user-agent string
:rtype: str
"""
if extras is None:
extras = []
return UserAgentBuilder(
name, version
).include_extras(
extras
).include_implementation(
).include_system().build()
class UserAgentBuilder(object):
"""Class to provide a greater level of control than :func:`user_agent`.
This is used by :func:`user_agent` to build its User-Agent string.
.. code-block:: python
user_agent_str = UserAgentBuilder(
name='requests-toolbelt',
version='17.4.0',
).include_implementation(
).include_system(
).include_extras([
('requests', '2.14.2'),
('urllib3', '1.21.2'),
]).build()
"""
format_string = '%s/%s'
def __init__(self, name, version):
"""Initialize our builder with the name and version of our user agent.
:param str name:
Name of our user-agent.
:param str version:
The version string for user-agent.
"""
self._pieces = collections.deque([(name, version)])
def build(self):
"""Finalize the User-Agent string.
:returns:
Formatted User-Agent string.
:rtype:
str
"""
return " ".join([self.format_string % piece for piece in self._pieces])
def include_extras(self, extras):
"""Include extra portions of the User-Agent.
:param list extras:
list of tuples of extra-name and extra-version
"""
if any(len(extra) != 2 for extra in extras):
raise ValueError('Extras should be a sequence of two item tuples.')
self._pieces.extend(extras)
return self
def include_implementation(self):
"""Append the implementation string to the user-agent string.
This adds the the information that you're using CPython 2.7.13 to the
User-Agent.
"""
self._pieces.append(_implementation_tuple())
return self
def include_system(self):
"""Append the information about the Operating System."""
self._pieces.append(_platform_tuple())
return self
def _implementation_tuple():
"""Return the tuple of interpreter name and version.
Returns a string that provides both the name and the version of the Python
implementation currently running. For example, on CPython 2.7.5 it will
return "CPython/2.7.5".
This function works best on CPython and PyPy: in particular, it probably
doesn't work for Jython or IronPython. Future investigation should be done
to work out the correct shape of the code for those platforms.
"""
implementation = platform.python_implementation()
if implementation == 'CPython':
implementation_version = platform.python_version()
elif implementation == 'PyPy':
implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major,
sys.pypy_version_info.minor,
sys.pypy_version_info.micro)
if sys.pypy_version_info.releaselevel != 'final':
implementation_version = ''.join([
implementation_version, sys.pypy_version_info.releaselevel
])
elif implementation == 'Jython':
implementation_version = platform.python_version() # Complete Guess
elif implementation == 'IronPython':
implementation_version = platform.python_version() # Complete Guess
else:
implementation_version = 'Unknown'
return (implementation, implementation_version)
def _implementation_string():
return "%s/%s" % _implementation_tuple()
def _platform_tuple():
try:
p_system = platform.system()
p_release = platform.release()
except IOError:
p_system = 'Unknown'
p_release = 'Unknown'
return (p_system, p_release)