diff --git a/channels/seriehd.py b/channels/seriehd.py index 0da31041..66a8cab5 100644 --- a/channels/seriehd.py +++ b/channels/seriehd.py @@ -33,7 +33,7 @@ def mainlist(item): @support.scrape def peliculas(item): #findhost() - debug=True + # debug=True patron = r'

(?P.*?)</h2>\s*<img src="(?P<thumb>[^"]+)" alt="[^"]*" />\s*<A HREF="(?P<url>[^"]+)">.*?<span class="year">(?:(?P<year>[0-9]{4}))?.*?<span class="calidad">(?:(?P<quality>[A-Z]+))?.*?</span>' patronNext=r'<span class="current">\d+</span><a rel="nofollow" class="page larger" href="([^"]+)">\d+</a>' action='episodios' diff --git a/default.py b/default.py index 6d810948..314f38af 100644 --- a/default.py +++ b/default.py @@ -8,6 +8,8 @@ import sys import xbmc from platformcode import config, logger +import ssl +logger.info(ssl.OPENSSL_VERSION) logger.info("init...") diff --git a/lib/requests/__init__.py b/lib/requests/__init__.py index a5b3c9c3..9a899df6 100644 --- a/lib/requests/__init__.py +++ b/lib/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.25 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 25 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -123,12 +123,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/lib/requests/__version__.py b/lib/requests/__version__.py index ef61ec0f..9844f740 100644 --- a/lib/requests/__version__.py +++ b/lib/requests/__version__.py @@ -5,10 +5,10 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.22.0' +__build__ = 0x022200 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2018 Kenneth Reitz' +__copyright__ = 'Copyright 2019 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/lib/requests/adapters.py b/lib/requests/adapters.py index a4b02842..fa4d9b3c 100644 --- a/lib/requests/adapters.py +++ b/lib/requests/adapters.py @@ -26,6 +26,7 @@ from urllib3.exceptions import ProtocolError from urllib3.exceptions import ReadTimeoutError from urllib3.exceptions import SSLError as _SSLError from urllib3.exceptions import ResponseError +from urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response <requests.Response>` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/lib/requests/api.py b/lib/requests/api.py index a2cc84d7..ef71d075 100644 --- a/lib/requests/api.py +++ b/lib/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the query string for the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') <Response [200]> """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object diff --git a/lib/requests/auth.py b/lib/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/lib/requests/auth.py +++ b/lib/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/lib/requests/compat.py b/lib/requests/compat.py index 6b9c6fac..c44b35ef 100644 --- a/lib/requests/compat.py +++ b/lib/requests/compat.py @@ -43,9 +43,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/lib/requests/cookies.py b/lib/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/lib/requests/cookies.py +++ b/lib/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/lib/requests/help.py b/lib/requests/help.py index 06e06b2a..e53d35ef 100644 --- a/lib/requests/help.py +++ b/lib/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/lib/requests/hooks.py b/lib/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/lib/requests/hooks.py +++ b/lib/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/lib/requests/models.py b/lib/requests/models.py index 3d0e1f42..62dcd0b7 100644 --- a/lib/requests/models.py +++ b/lib/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() <PreparedRequest [GET]> """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() <PreparedRequest [GET]> @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): @@ -780,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/lib/requests/sessions.py b/lib/requests/sessions.py index ba135268..d73d700f 100644 --- a/lib/requests/sessions.py +++ b/lib/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -115,6 +115,31 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + + # Standard case: root URI must match + return changed_port or changed_scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +261,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +320,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +346,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') <Response [200]> Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') <Response [200]> """ @@ -453,8 +474,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +571,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +584,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +596,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +747,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +759,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/lib/requests/status_codes.py b/lib/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/lib/requests/status_codes.py +++ b/lib/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/lib/requests/utils.py b/lib/requests/utils.py index 431f6be0..8170a8d2 100644 --- a/lib/requests/utils.py +++ b/lib/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -173,10 +175,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -466,7 +468,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +708,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +731,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +739,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/lib/tornado/__init__.py b/lib/tornado/__init__.py deleted file mode 100755 index b269cf70..00000000 --- a/lib/tornado/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""The Tornado web server and tools.""" - -from __future__ import absolute_import, division, print_function - -# version is a human-readable version number. - -# version_info is a four-tuple for programmatic comparison. The first -# three numbers are the components of the version number. The fourth -# is zero for an official release, positive for a development branch, -# or negative for a release candidate or beta (after the base version -# number has been incremented) -version = "5.1.1" -version_info = (5, 1, 1, 0) diff --git a/lib/tornado/_locale_data.py b/lib/tornado/_locale_data.py deleted file mode 100755 index a2c50390..00000000 --- a/lib/tornado/_locale_data.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Data used by the tornado.locale module.""" - -from __future__ import absolute_import, division, print_function - -LOCALE_NAMES = { - "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"}, - "am_ET": {"name_en": u"Amharic", "name": u"አማርኛ"}, - "ar_AR": {"name_en": u"Arabic", "name": u"العربية"}, - "bg_BG": {"name_en": u"Bulgarian", "name": u"Български"}, - "bn_IN": {"name_en": u"Bengali", "name": u"বাংলা"}, - "bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"}, - "ca_ES": {"name_en": u"Catalan", "name": u"Català"}, - "cs_CZ": {"name_en": u"Czech", "name": u"Čeština"}, - "cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"}, - "da_DK": {"name_en": u"Danish", "name": u"Dansk"}, - "de_DE": {"name_en": u"German", "name": u"Deutsch"}, - "el_GR": {"name_en": u"Greek", "name": u"Ελληνικά"}, - "en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"}, - "en_US": {"name_en": u"English (US)", "name": u"English (US)"}, - "es_ES": {"name_en": u"Spanish (Spain)", "name": u"Español (España)"}, - "es_LA": {"name_en": u"Spanish", "name": u"Español"}, - "et_EE": {"name_en": u"Estonian", "name": u"Eesti"}, - "eu_ES": {"name_en": u"Basque", "name": u"Euskara"}, - "fa_IR": {"name_en": u"Persian", "name": u"فارسی"}, - "fi_FI": {"name_en": u"Finnish", "name": u"Suomi"}, - "fr_CA": {"name_en": u"French (Canada)", "name": u"Français (Canada)"}, - "fr_FR": {"name_en": u"French", "name": u"Français"}, - "ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"}, - "gl_ES": {"name_en": u"Galician", "name": u"Galego"}, - "he_IL": {"name_en": u"Hebrew", "name": u"עברית"}, - "hi_IN": {"name_en": u"Hindi", "name": u"हिन्दी"}, - "hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"}, - "hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"}, - "id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"}, - "is_IS": {"name_en": u"Icelandic", "name": u"Íslenska"}, - "it_IT": {"name_en": u"Italian", "name": u"Italiano"}, - "ja_JP": {"name_en": u"Japanese", "name": u"日本語"}, - "ko_KR": {"name_en": u"Korean", "name": u"한국어"}, - "lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvių"}, - "lv_LV": {"name_en": u"Latvian", "name": u"Latviešu"}, - "mk_MK": {"name_en": u"Macedonian", "name": u"Македонски"}, - "ml_IN": {"name_en": u"Malayalam", "name": u"മലയാളം"}, - "ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"}, - "nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokmål)"}, - "nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"}, - "nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"}, - "pa_IN": {"name_en": u"Punjabi", "name": u"ਪੰਜਾਬੀ"}, - "pl_PL": {"name_en": u"Polish", "name": u"Polski"}, - "pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Português (Brasil)"}, - "pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Português (Portugal)"}, - "ro_RO": {"name_en": u"Romanian", "name": u"Română"}, - "ru_RU": {"name_en": u"Russian", "name": u"Русский"}, - "sk_SK": {"name_en": u"Slovak", "name": u"Slovenčina"}, - "sl_SI": {"name_en": u"Slovenian", "name": u"Slovenščina"}, - "sq_AL": {"name_en": u"Albanian", "name": u"Shqip"}, - "sr_RS": {"name_en": u"Serbian", "name": u"Српски"}, - "sv_SE": {"name_en": u"Swedish", "name": u"Svenska"}, - "sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"}, - "ta_IN": {"name_en": u"Tamil", "name": u"தமிழ்"}, - "te_IN": {"name_en": u"Telugu", "name": u"తెలుగు"}, - "th_TH": {"name_en": u"Thai", "name": u"ภาษาไทย"}, - "tl_PH": {"name_en": u"Filipino", "name": u"Filipino"}, - "tr_TR": {"name_en": u"Turkish", "name": u"Türkçe"}, - "uk_UA": {"name_en": u"Ukraini ", "name": u"Українська"}, - "vi_VN": {"name_en": u"Vietnamese", "name": u"Tiếng Việt"}, - "zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"中文(简体)"}, - "zh_TW": {"name_en": u"Chinese (Traditional)", "name": u"中文(繁體)"}, -} diff --git a/lib/tornado/auth.py b/lib/tornado/auth.py deleted file mode 100755 index b79ad14b..00000000 --- a/lib/tornado/auth.py +++ /dev/null @@ -1,1236 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""This module contains implementations of various third-party -authentication schemes. - -All the classes in this file are class mixins designed to be used with -the `tornado.web.RequestHandler` class. They are used in two ways: - -* On a login handler, use methods such as ``authenticate_redirect()``, - ``authorize_redirect()``, and ``get_authenticated_user()`` to - establish the user's identity and store authentication tokens to your - database and/or cookies. -* In non-login handlers, use methods such as ``facebook_request()`` - or ``twitter_request()`` to use the authentication tokens to make - requests to the respective services. - -They all take slightly different arguments due to the fact all these -services implement authentication and authorization slightly differently. -See the individual service classes below for complete documentation. - -Example usage for Google OAuth: - -.. testcode:: - - class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleOAuth2Mixin): - async def get(self): - if self.get_argument('code', False): - user = await self.get_authenticated_user( - redirect_uri='http://your.site.com/auth/google', - code=self.get_argument('code')) - # Save the user with e.g. set_secure_cookie - else: - await self.authorize_redirect( - redirect_uri='http://your.site.com/auth/google', - client_id=self.settings['google_oauth']['key'], - scope=['profile', 'email'], - response_type='code', - extra_params={'approval_prompt': 'auto'}) - -.. testoutput:: - :hide: - - -.. versionchanged:: 4.0 - All of the callback interfaces in this module are now guaranteed - to run their callback with an argument of ``None`` on error. - Previously some functions would do this while others would simply - terminate the request on their own. This change also ensures that - errors are more consistently reported through the ``Future`` interfaces. -""" - -from __future__ import absolute_import, division, print_function - -import base64 -import binascii -import functools -import hashlib -import hmac -import time -import uuid -import warnings - -from tornado.concurrent import (Future, _non_deprecated_return_future, - future_set_exc_info, chain_future, - future_set_result_unless_cancelled) -from tornado import gen -from tornado import httpclient -from tornado import escape -from tornado.httputil import url_concat -from tornado.log import gen_log -from tornado.stack_context import ExceptionStackContext, wrap -from tornado.util import unicode_type, ArgReplacer, PY3 - -if PY3: - import urllib.parse as urlparse - import urllib.parse as urllib_parse - long = int -else: - import urlparse - import urllib as urllib_parse - - -class AuthError(Exception): - pass - - -def _auth_future_to_callback(callback, future): - try: - result = future.result() - except AuthError as e: - gen_log.warning(str(e)) - result = None - callback(result) - - -def _auth_return_future(f): - """Similar to tornado.concurrent.return_future, but uses the auth - module's legacy callback interface. - - Note that when using this decorator the ``callback`` parameter - inside the function will actually be a future. - - .. deprecated:: 5.1 - Will be removed in 6.0. - """ - replacer = ArgReplacer(f, 'callback') - - @functools.wraps(f) - def wrapper(*args, **kwargs): - future = Future() - callback, args, kwargs = replacer.replace(future, args, kwargs) - if callback is not None: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - future.add_done_callback( - wrap(functools.partial(_auth_future_to_callback, callback))) - - def handle_exception(typ, value, tb): - if future.done(): - return False - else: - future_set_exc_info(future, (typ, value, tb)) - return True - with ExceptionStackContext(handle_exception, delay_warning=True): - f(*args, **kwargs) - return future - return wrapper - - -class OpenIdMixin(object): - """Abstract implementation of OpenID and Attribute Exchange. - - Class attributes: - - * ``_OPENID_ENDPOINT``: the identity provider's URI. - """ - @_non_deprecated_return_future - def authenticate_redirect(self, callback_uri=None, - ax_attrs=["name", "email", "language", "username"], - callback=None): - """Redirects to the authentication URL for this service. - - After authentication, the service will redirect back to the given - callback URI with additional parameters including ``openid.mode``. - - We request the given attributes for the authenticated user by - default (name, email, language, and username). If you don't need - all those attributes for your app, you can request fewer with - the ax_attrs keyword argument. - - .. versionchanged:: 3.1 - Returns a `.Future` and takes an optional callback. These are - not strictly necessary as this method is synchronous, - but they are supplied for consistency with - `OAuthMixin.authorize_redirect`. - - .. deprecated:: 5.1 - - The ``callback`` argument and returned awaitable will be removed - in Tornado 6.0; this will be an ordinary synchronous function. - """ - callback_uri = callback_uri or self.request.uri - args = self._openid_args(callback_uri, ax_attrs=ax_attrs) - self.redirect(self._OPENID_ENDPOINT + "?" + urllib_parse.urlencode(args)) - callback() - - @_auth_return_future - def get_authenticated_user(self, callback, http_client=None): - """Fetches the authenticated user data upon redirect. - - This method should be called by the handler that receives the - redirect from the `authenticate_redirect()` method (which is - often the same as the one that calls it; in that case you would - call `get_authenticated_user` if the ``openid.mode`` parameter - is present and `authenticate_redirect` if it is not). - - The result of this method will generally be used to set a cookie. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - # Verify the OpenID response via direct request to the OP - args = dict((k, v[-1]) for k, v in self.request.arguments.items()) - args["openid.mode"] = u"check_authentication" - url = self._OPENID_ENDPOINT - if http_client is None: - http_client = self.get_auth_http_client() - fut = http_client.fetch(url, method="POST", body=urllib_parse.urlencode(args)) - fut.add_done_callback(wrap(functools.partial( - self._on_authentication_verified, callback))) - - def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None): - url = urlparse.urljoin(self.request.full_url(), callback_uri) - args = { - "openid.ns": "http://specs.openid.net/auth/2.0", - "openid.claimed_id": - "http://specs.openid.net/auth/2.0/identifier_select", - "openid.identity": - "http://specs.openid.net/auth/2.0/identifier_select", - "openid.return_to": url, - "openid.realm": urlparse.urljoin(url, '/'), - "openid.mode": "checkid_setup", - } - if ax_attrs: - args.update({ - "openid.ns.ax": "http://openid.net/srv/ax/1.0", - "openid.ax.mode": "fetch_request", - }) - ax_attrs = set(ax_attrs) - required = [] - if "name" in ax_attrs: - ax_attrs -= set(["name", "firstname", "fullname", "lastname"]) - required += ["firstname", "fullname", "lastname"] - args.update({ - "openid.ax.type.firstname": - "http://axschema.org/namePerson/first", - "openid.ax.type.fullname": - "http://axschema.org/namePerson", - "openid.ax.type.lastname": - "http://axschema.org/namePerson/last", - }) - known_attrs = { - "email": "http://axschema.org/contact/email", - "language": "http://axschema.org/pref/language", - "username": "http://axschema.org/namePerson/friendly", - } - for name in ax_attrs: - args["openid.ax.type." + name] = known_attrs[name] - required.append(name) - args["openid.ax.required"] = ",".join(required) - if oauth_scope: - args.update({ - "openid.ns.oauth": - "http://specs.openid.net/extensions/oauth/1.0", - "openid.oauth.consumer": self.request.host.split(":")[0], - "openid.oauth.scope": oauth_scope, - }) - return args - - def _on_authentication_verified(self, future, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError( - "Error response %s" % e)) - return - if b"is_valid:true" not in response.body: - future.set_exception(AuthError( - "Invalid OpenID response: %s" % response.body)) - return - - # Make sure we got back at least an email from attribute exchange - ax_ns = None - for name in self.request.arguments: - if name.startswith("openid.ns.") and \ - self.get_argument(name) == u"http://openid.net/srv/ax/1.0": - ax_ns = name[10:] - break - - def get_ax_arg(uri): - if not ax_ns: - return u"" - prefix = "openid." + ax_ns + ".type." - ax_name = None - for name in self.request.arguments.keys(): - if self.get_argument(name) == uri and name.startswith(prefix): - part = name[len(prefix):] - ax_name = "openid." + ax_ns + ".value." + part - break - if not ax_name: - return u"" - return self.get_argument(ax_name, u"") - - email = get_ax_arg("http://axschema.org/contact/email") - name = get_ax_arg("http://axschema.org/namePerson") - first_name = get_ax_arg("http://axschema.org/namePerson/first") - last_name = get_ax_arg("http://axschema.org/namePerson/last") - username = get_ax_arg("http://axschema.org/namePerson/friendly") - locale = get_ax_arg("http://axschema.org/pref/language").lower() - user = dict() - name_parts = [] - if first_name: - user["first_name"] = first_name - name_parts.append(first_name) - if last_name: - user["last_name"] = last_name - name_parts.append(last_name) - if name: - user["name"] = name - elif name_parts: - user["name"] = u" ".join(name_parts) - elif email: - user["name"] = email.split("@")[0] - if email: - user["email"] = email - if locale: - user["locale"] = locale - if username: - user["username"] = username - claimed_id = self.get_argument("openid.claimed_id", None) - if claimed_id: - user["claimed_id"] = claimed_id - future_set_result_unless_cancelled(future, user) - - def get_auth_http_client(self): - """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - - May be overridden by subclasses to use an HTTP client other than - the default. - """ - return httpclient.AsyncHTTPClient() - - -class OAuthMixin(object): - """Abstract implementation of OAuth 1.0 and 1.0a. - - See `TwitterMixin` below for an example implementation. - - Class attributes: - - * ``_OAUTH_AUTHORIZE_URL``: The service's OAuth authorization url. - * ``_OAUTH_ACCESS_TOKEN_URL``: The service's OAuth access token url. - * ``_OAUTH_VERSION``: May be either "1.0" or "1.0a". - * ``_OAUTH_NO_CALLBACKS``: Set this to True if the service requires - advance registration of callbacks. - - Subclasses must also override the `_oauth_get_user_future` and - `_oauth_consumer_token` methods. - """ - @_non_deprecated_return_future - def authorize_redirect(self, callback_uri=None, extra_params=None, - http_client=None, callback=None): - """Redirects the user to obtain OAuth authorization for this service. - - The ``callback_uri`` may be omitted if you have previously - registered a callback URI with the third-party service. For - some services, you must use a previously-registered callback - URI and cannot specify a callback via this method. - - This method sets a cookie called ``_oauth_request_token`` which is - subsequently used (and cleared) in `get_authenticated_user` for - security purposes. - - This method is asynchronous and must be called with ``await`` - or ``yield`` (This is different from other ``auth*_redirect`` - methods defined in this module). It calls - `.RequestHandler.finish` for you so you should not write any - other response after it returns. - - .. versionchanged:: 3.1 - Now returns a `.Future` and takes an optional callback, for - compatibility with `.gen.coroutine`. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - - """ - if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False): - raise Exception("This service does not support oauth_callback") - if http_client is None: - http_client = self.get_auth_http_client() - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - fut = http_client.fetch( - self._oauth_request_token_url(callback_uri=callback_uri, - extra_params=extra_params)) - fut.add_done_callback(wrap(functools.partial( - self._on_request_token, - self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback))) - else: - fut = http_client.fetch(self._oauth_request_token_url()) - fut.add_done_callback( - wrap(functools.partial( - self._on_request_token, self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback))) - - @_auth_return_future - def get_authenticated_user(self, callback, http_client=None): - """Gets the OAuth authorized user and access token. - - This method should be called from the handler for your - OAuth callback URL to complete the registration process. We run the - callback with the authenticated user dictionary. This dictionary - will contain an ``access_key`` which can be used to make authorized - requests to this service on behalf of the user. The dictionary will - also contain other fields such as ``name``, depending on the service - used. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - future = callback - request_key = escape.utf8(self.get_argument("oauth_token")) - oauth_verifier = self.get_argument("oauth_verifier", None) - request_cookie = self.get_cookie("_oauth_request_token") - if not request_cookie: - future.set_exception(AuthError( - "Missing OAuth request token cookie")) - return - self.clear_cookie("_oauth_request_token") - cookie_key, cookie_secret = [ - base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")] - if cookie_key != request_key: - future.set_exception(AuthError( - "Request token does not match cookie")) - return - token = dict(key=cookie_key, secret=cookie_secret) - if oauth_verifier: - token["verifier"] = oauth_verifier - if http_client is None: - http_client = self.get_auth_http_client() - fut = http_client.fetch(self._oauth_access_token_url(token)) - fut.add_done_callback(wrap(functools.partial(self._on_access_token, callback))) - - def _oauth_request_token_url(self, callback_uri=None, extra_params=None): - consumer_token = self._oauth_consumer_token() - url = self._OAUTH_REQUEST_TOKEN_URL - args = dict( - oauth_consumer_key=escape.to_basestring(consumer_token["key"]), - oauth_signature_method="HMAC-SHA1", - oauth_timestamp=str(int(time.time())), - oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version="1.0", - ) - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - if callback_uri == "oob": - args["oauth_callback"] = "oob" - elif callback_uri: - args["oauth_callback"] = urlparse.urljoin( - self.request.full_url(), callback_uri) - if extra_params: - args.update(extra_params) - signature = _oauth10a_signature(consumer_token, "GET", url, args) - else: - signature = _oauth_signature(consumer_token, "GET", url, args) - - args["oauth_signature"] = signature - return url + "?" + urllib_parse.urlencode(args) - - def _on_request_token(self, authorize_url, callback_uri, callback, - response_fut): - try: - response = response_fut.result() - except Exception as e: - raise Exception("Could not get request token: %s" % e) - request_token = _oauth_parse_response(response.body) - data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" + - base64.b64encode(escape.utf8(request_token["secret"]))) - self.set_cookie("_oauth_request_token", data) - args = dict(oauth_token=request_token["key"]) - if callback_uri == "oob": - self.finish(authorize_url + "?" + urllib_parse.urlencode(args)) - callback() - return - elif callback_uri: - args["oauth_callback"] = urlparse.urljoin( - self.request.full_url(), callback_uri) - self.redirect(authorize_url + "?" + urllib_parse.urlencode(args)) - callback() - - def _oauth_access_token_url(self, request_token): - consumer_token = self._oauth_consumer_token() - url = self._OAUTH_ACCESS_TOKEN_URL - args = dict( - oauth_consumer_key=escape.to_basestring(consumer_token["key"]), - oauth_token=escape.to_basestring(request_token["key"]), - oauth_signature_method="HMAC-SHA1", - oauth_timestamp=str(int(time.time())), - oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version="1.0", - ) - if "verifier" in request_token: - args["oauth_verifier"] = request_token["verifier"] - - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - signature = _oauth10a_signature(consumer_token, "GET", url, args, - request_token) - else: - signature = _oauth_signature(consumer_token, "GET", url, args, - request_token) - - args["oauth_signature"] = signature - return url + "?" + urllib_parse.urlencode(args) - - def _on_access_token(self, future, response_fut): - try: - response = response_fut.result() - except Exception: - future.set_exception(AuthError("Could not fetch access token")) - return - - access_token = _oauth_parse_response(response.body) - fut = self._oauth_get_user_future(access_token) - fut = gen.convert_yielded(fut) - fut.add_done_callback( - wrap(functools.partial(self._on_oauth_get_user, access_token, future))) - - def _oauth_consumer_token(self): - """Subclasses must override this to return their OAuth consumer keys. - - The return value should be a `dict` with keys ``key`` and ``secret``. - """ - raise NotImplementedError() - - @_non_deprecated_return_future - def _oauth_get_user_future(self, access_token, callback): - """Subclasses must override this to get basic information about the - user. - - Should return a `.Future` whose result is a dictionary - containing information about the user, which may have been - retrieved by using ``access_token`` to make a request to the - service. - - The access token will be added to the returned dictionary to make - the result of `get_authenticated_user`. - - For backwards compatibility, the callback-based ``_oauth_get_user`` - method is also supported. - - .. versionchanged:: 5.1 - - Subclasses may also define this method with ``async def``. - - .. deprecated:: 5.1 - - The ``_oauth_get_user`` fallback is deprecated and support for it - will be removed in 6.0. - """ - warnings.warn("_oauth_get_user is deprecated, override _oauth_get_user_future instead", - DeprecationWarning) - # By default, call the old-style _oauth_get_user, but new code - # should override this method instead. - self._oauth_get_user(access_token, callback) - - def _oauth_get_user(self, access_token, callback): - raise NotImplementedError() - - def _on_oauth_get_user(self, access_token, future, user_future): - if user_future.exception() is not None: - future.set_exception(user_future.exception()) - return - user = user_future.result() - if not user: - future.set_exception(AuthError("Error getting user")) - return - user["access_token"] = access_token - future_set_result_unless_cancelled(future, user) - - def _oauth_request_parameters(self, url, access_token, parameters={}, - method="GET"): - """Returns the OAuth parameters as a dict for the given request. - - parameters should include all POST arguments and query string arguments - that will be sent with the request. - """ - consumer_token = self._oauth_consumer_token() - base_args = dict( - oauth_consumer_key=escape.to_basestring(consumer_token["key"]), - oauth_token=escape.to_basestring(access_token["key"]), - oauth_signature_method="HMAC-SHA1", - oauth_timestamp=str(int(time.time())), - oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), - oauth_version="1.0", - ) - args = {} - args.update(base_args) - args.update(parameters) - if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - signature = _oauth10a_signature(consumer_token, method, url, args, - access_token) - else: - signature = _oauth_signature(consumer_token, method, url, args, - access_token) - base_args["oauth_signature"] = escape.to_basestring(signature) - return base_args - - def get_auth_http_client(self): - """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - - May be overridden by subclasses to use an HTTP client other than - the default. - """ - return httpclient.AsyncHTTPClient() - - -class OAuth2Mixin(object): - """Abstract implementation of OAuth 2.0. - - See `FacebookGraphMixin` or `GoogleOAuth2Mixin` below for example - implementations. - - Class attributes: - - * ``_OAUTH_AUTHORIZE_URL``: The service's authorization url. - * ``_OAUTH_ACCESS_TOKEN_URL``: The service's access token url. - """ - @_non_deprecated_return_future - def authorize_redirect(self, redirect_uri=None, client_id=None, - client_secret=None, extra_params=None, - callback=None, scope=None, response_type="code"): - """Redirects the user to obtain OAuth authorization for this service. - - Some providers require that you register a redirect URL with - your application instead of passing one via this method. You - should call this method to log the user in, and then call - ``get_authenticated_user`` in the handler for your - redirect URL to complete the authorization process. - - .. versionchanged:: 3.1 - Returns a `.Future` and takes an optional callback. These are - not strictly necessary as this method is synchronous, - but they are supplied for consistency with - `OAuthMixin.authorize_redirect`. - - .. deprecated:: 5.1 - - The ``callback`` argument and returned awaitable will be removed - in Tornado 6.0; this will be an ordinary synchronous function. - """ - args = { - "redirect_uri": redirect_uri, - "client_id": client_id, - "response_type": response_type - } - if extra_params: - args.update(extra_params) - if scope: - args['scope'] = ' '.join(scope) - self.redirect( - url_concat(self._OAUTH_AUTHORIZE_URL, args)) - callback() - - def _oauth_request_token_url(self, redirect_uri=None, client_id=None, - client_secret=None, code=None, - extra_params=None): - url = self._OAUTH_ACCESS_TOKEN_URL - args = dict( - redirect_uri=redirect_uri, - code=code, - client_id=client_id, - client_secret=client_secret, - ) - if extra_params: - args.update(extra_params) - return url_concat(url, args) - - @_auth_return_future - def oauth2_request(self, url, callback, access_token=None, - post_args=None, **args): - """Fetches the given URL auth an OAuth2 access token. - - If the request is a POST, ``post_args`` should be provided. Query - string arguments should be given as keyword arguments. - - Example usage: - - ..testcode:: - - class MainHandler(tornado.web.RequestHandler, - tornado.auth.FacebookGraphMixin): - @tornado.web.authenticated - async def get(self): - new_entry = await self.oauth2_request( - "https://graph.facebook.com/me/feed", - post_args={"message": "I am posting from my Tornado application!"}, - access_token=self.current_user["access_token"]) - - if not new_entry: - # Call failed; perhaps missing permission? - await self.authorize_redirect() - return - self.finish("Posted a message!") - - .. testoutput:: - :hide: - - .. versionadded:: 4.3 - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - all_args = {} - if access_token: - all_args["access_token"] = access_token - all_args.update(args) - - if all_args: - url += "?" + urllib_parse.urlencode(all_args) - callback = wrap(functools.partial(self._on_oauth2_request, callback)) - http = self.get_auth_http_client() - if post_args is not None: - fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) - else: - fut = http.fetch(url) - fut.add_done_callback(callback) - - def _on_oauth2_request(self, future, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError("Error response %s" % e)) - return - - future_set_result_unless_cancelled(future, escape.json_decode(response.body)) - - def get_auth_http_client(self): - """Returns the `.AsyncHTTPClient` instance to be used for auth requests. - - May be overridden by subclasses to use an HTTP client other than - the default. - - .. versionadded:: 4.3 - """ - return httpclient.AsyncHTTPClient() - - -class TwitterMixin(OAuthMixin): - """Twitter OAuth authentication. - - To authenticate with Twitter, register your application with - Twitter at http://twitter.com/apps. Then copy your Consumer Key - and Consumer Secret to the application - `~tornado.web.Application.settings` ``twitter_consumer_key`` and - ``twitter_consumer_secret``. Use this mixin on the handler for the - URL you registered as your application's callback URL. - - When your application is set up, you can use this mixin like this - to authenticate the user with Twitter and get access to their stream: - - .. testcode:: - - class TwitterLoginHandler(tornado.web.RequestHandler, - tornado.auth.TwitterMixin): - async def get(self): - if self.get_argument("oauth_token", None): - user = await self.get_authenticated_user() - # Save the user using e.g. set_secure_cookie() - else: - await self.authorize_redirect() - - .. testoutput:: - :hide: - - The user object returned by `~OAuthMixin.get_authenticated_user` - includes the attributes ``username``, ``name``, ``access_token``, - and all of the custom Twitter user attributes described at - https://dev.twitter.com/docs/api/1.1/get/users/show - """ - _OAUTH_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" - _OAUTH_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" - _OAUTH_AUTHORIZE_URL = "https://api.twitter.com/oauth/authorize" - _OAUTH_AUTHENTICATE_URL = "https://api.twitter.com/oauth/authenticate" - _OAUTH_NO_CALLBACKS = False - _TWITTER_BASE_URL = "https://api.twitter.com/1.1" - - @_non_deprecated_return_future - def authenticate_redirect(self, callback_uri=None, callback=None): - """Just like `~OAuthMixin.authorize_redirect`, but - auto-redirects if authorized. - - This is generally the right interface to use if you are using - Twitter for single-sign on. - - .. versionchanged:: 3.1 - Now returns a `.Future` and takes an optional callback, for - compatibility with `.gen.coroutine`. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - http = self.get_auth_http_client() - fut = http.fetch(self._oauth_request_token_url(callback_uri=callback_uri)) - fut.add_done_callback(wrap(functools.partial( - self._on_request_token, self._OAUTH_AUTHENTICATE_URL, - None, callback))) - - @_auth_return_future - def twitter_request(self, path, callback=None, access_token=None, - post_args=None, **args): - """Fetches the given API path, e.g., ``statuses/user_timeline/btaylor`` - - The path should not include the format or API version number. - (we automatically use JSON format and API version 1). - - If the request is a POST, ``post_args`` should be provided. Query - string arguments should be given as keyword arguments. - - All the Twitter methods are documented at http://dev.twitter.com/ - - Many methods require an OAuth access token which you can - obtain through `~OAuthMixin.authorize_redirect` and - `~OAuthMixin.get_authenticated_user`. The user returned through that - process includes an 'access_token' attribute that can be used - to make authenticated requests via this method. Example - usage: - - .. testcode:: - - class MainHandler(tornado.web.RequestHandler, - tornado.auth.TwitterMixin): - @tornado.web.authenticated - async def get(self): - new_entry = await self.twitter_request( - "/statuses/update", - post_args={"status": "Testing Tornado Web Server"}, - access_token=self.current_user["access_token"]) - if not new_entry: - # Call failed; perhaps missing permission? - yield self.authorize_redirect() - return - self.finish("Posted a message!") - - .. testoutput:: - :hide: - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - if path.startswith('http:') or path.startswith('https:'): - # Raw urls are useful for e.g. search which doesn't follow the - # usual pattern: http://search.twitter.com/search.json - url = path - else: - url = self._TWITTER_BASE_URL + path + ".json" - # Add the OAuth resource request signature if we have credentials - if access_token: - all_args = {} - all_args.update(args) - all_args.update(post_args or {}) - method = "POST" if post_args is not None else "GET" - oauth = self._oauth_request_parameters( - url, access_token, all_args, method=method) - args.update(oauth) - if args: - url += "?" + urllib_parse.urlencode(args) - http = self.get_auth_http_client() - http_callback = wrap(functools.partial(self._on_twitter_request, callback, url)) - if post_args is not None: - fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) - else: - fut = http.fetch(url) - fut.add_done_callback(http_callback) - - def _on_twitter_request(self, future, url, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError( - "Error response %s fetching %s" % (e, url))) - return - future_set_result_unless_cancelled(future, escape.json_decode(response.body)) - - def _oauth_consumer_token(self): - self.require_setting("twitter_consumer_key", "Twitter OAuth") - self.require_setting("twitter_consumer_secret", "Twitter OAuth") - return dict( - key=self.settings["twitter_consumer_key"], - secret=self.settings["twitter_consumer_secret"]) - - @gen.coroutine - def _oauth_get_user_future(self, access_token): - user = yield self.twitter_request( - "/account/verify_credentials", - access_token=access_token) - if user: - user["username"] = user["screen_name"] - raise gen.Return(user) - - -class GoogleOAuth2Mixin(OAuth2Mixin): - """Google authentication using OAuth2. - - In order to use, register your application with Google and copy the - relevant parameters to your application settings. - - * Go to the Google Dev Console at http://console.developers.google.com - * Select a project, or create a new one. - * In the sidebar on the left, select APIs & Auth. - * In the list of APIs, find the Google+ API service and set it to ON. - * In the sidebar on the left, select Credentials. - * In the OAuth section of the page, select Create New Client ID. - * Set the Redirect URI to point to your auth handler - * Copy the "Client secret" and "Client ID" to the application settings as - {"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}} - - .. versionadded:: 3.2 - """ - _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/v2/auth" - _OAUTH_ACCESS_TOKEN_URL = "https://www.googleapis.com/oauth2/v4/token" - _OAUTH_USERINFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo" - _OAUTH_NO_CALLBACKS = False - _OAUTH_SETTINGS_KEY = 'google_oauth' - - @_auth_return_future - def get_authenticated_user(self, redirect_uri, code, callback): - """Handles the login for the Google user, returning an access token. - - The result is a dictionary containing an ``access_token`` field - ([among others](https://developers.google.com/identity/protocols/OAuth2WebServer#handlingtheresponse)). - Unlike other ``get_authenticated_user`` methods in this package, - this method does not return any additional information about the user. - The returned access token can be used with `OAuth2Mixin.oauth2_request` - to request additional information (perhaps from - ``https://www.googleapis.com/oauth2/v2/userinfo``) - - Example usage: - - .. testcode:: - - class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleOAuth2Mixin): - async def get(self): - if self.get_argument('code', False): - access = await self.get_authenticated_user( - redirect_uri='http://your.site.com/auth/google', - code=self.get_argument('code')) - user = await self.oauth2_request( - "https://www.googleapis.com/oauth2/v1/userinfo", - access_token=access["access_token"]) - # Save the user and access token with - # e.g. set_secure_cookie. - else: - await self.authorize_redirect( - redirect_uri='http://your.site.com/auth/google', - client_id=self.settings['google_oauth']['key'], - scope=['profile', 'email'], - response_type='code', - extra_params={'approval_prompt': 'auto'}) - - .. testoutput:: - :hide: - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ # noqa: E501 - http = self.get_auth_http_client() - body = urllib_parse.urlencode({ - "redirect_uri": redirect_uri, - "code": code, - "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'], - "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'], - "grant_type": "authorization_code", - }) - - fut = http.fetch(self._OAUTH_ACCESS_TOKEN_URL, - method="POST", - headers={'Content-Type': 'application/x-www-form-urlencoded'}, - body=body) - fut.add_done_callback(wrap(functools.partial(self._on_access_token, callback))) - - def _on_access_token(self, future, response_fut): - """Callback function for the exchange to the access token.""" - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError('Google auth error: %s' % str(e))) - return - - args = escape.json_decode(response.body) - future_set_result_unless_cancelled(future, args) - - -class FacebookGraphMixin(OAuth2Mixin): - """Facebook authentication using the new Graph API and OAuth2.""" - _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?" - _OAUTH_AUTHORIZE_URL = "https://www.facebook.com/dialog/oauth?" - _OAUTH_NO_CALLBACKS = False - _FACEBOOK_BASE_URL = "https://graph.facebook.com" - - @_auth_return_future - def get_authenticated_user(self, redirect_uri, client_id, client_secret, - code, callback, extra_fields=None): - """Handles the login for the Facebook user, returning a user object. - - Example usage: - - .. testcode:: - - class FacebookGraphLoginHandler(tornado.web.RequestHandler, - tornado.auth.FacebookGraphMixin): - async def get(self): - if self.get_argument("code", False): - user = await self.get_authenticated_user( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - client_secret=self.settings["facebook_secret"], - code=self.get_argument("code")) - # Save the user with e.g. set_secure_cookie - else: - await self.authorize_redirect( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - extra_params={"scope": "read_stream,offline_access"}) - - .. testoutput:: - :hide: - - This method returns a dictionary which may contain the following fields: - - * ``access_token``, a string which may be passed to `facebook_request` - * ``session_expires``, an integer encoded as a string representing - the time until the access token expires in seconds. This field should - be used like ``int(user['session_expires'])``; in a future version of - Tornado it will change from a string to an integer. - * ``id``, ``name``, ``first_name``, ``last_name``, ``locale``, ``picture``, - ``link``, plus any fields named in the ``extra_fields`` argument. These - fields are copied from the Facebook graph API - `user object <https://developers.facebook.com/docs/graph-api/reference/user>`_ - - .. versionchanged:: 4.5 - The ``session_expires`` field was updated to support changes made to the - Facebook API in March 2017. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - http = self.get_auth_http_client() - args = { - "redirect_uri": redirect_uri, - "code": code, - "client_id": client_id, - "client_secret": client_secret, - } - - fields = set(['id', 'name', 'first_name', 'last_name', - 'locale', 'picture', 'link']) - if extra_fields: - fields.update(extra_fields) - - fut = http.fetch(self._oauth_request_token_url(**args)) - fut.add_done_callback(wrap(functools.partial(self._on_access_token, redirect_uri, client_id, - client_secret, callback, fields))) - - @gen.coroutine - def _on_access_token(self, redirect_uri, client_id, client_secret, - future, fields, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError('Facebook auth error: %s' % str(e))) - return - - args = escape.json_decode(response.body) - session = { - "access_token": args.get("access_token"), - "expires_in": args.get("expires_in") - } - - user = yield self.facebook_request( - path="/me", - access_token=session["access_token"], - appsecret_proof=hmac.new(key=client_secret.encode('utf8'), - msg=session["access_token"].encode('utf8'), - digestmod=hashlib.sha256).hexdigest(), - fields=",".join(fields) - ) - - if user is None: - future_set_result_unless_cancelled(future, None) - return - - fieldmap = {} - for field in fields: - fieldmap[field] = user.get(field) - - # session_expires is converted to str for compatibility with - # older versions in which the server used url-encoding and - # this code simply returned the string verbatim. - # This should change in Tornado 5.0. - fieldmap.update({"access_token": session["access_token"], - "session_expires": str(session.get("expires_in"))}) - future_set_result_unless_cancelled(future, fieldmap) - - @_auth_return_future - def facebook_request(self, path, callback, access_token=None, - post_args=None, **args): - """Fetches the given relative API path, e.g., "/btaylor/picture" - - If the request is a POST, ``post_args`` should be provided. Query - string arguments should be given as keyword arguments. - - An introduction to the Facebook Graph API can be found at - http://developers.facebook.com/docs/api - - Many methods require an OAuth access token which you can - obtain through `~OAuth2Mixin.authorize_redirect` and - `get_authenticated_user`. The user returned through that - process includes an ``access_token`` attribute that can be - used to make authenticated requests via this method. - - Example usage: - - .. testcode:: - - class MainHandler(tornado.web.RequestHandler, - tornado.auth.FacebookGraphMixin): - @tornado.web.authenticated - async def get(self): - new_entry = await self.facebook_request( - "/me/feed", - post_args={"message": "I am posting from my Tornado application!"}, - access_token=self.current_user["access_token"]) - - if not new_entry: - # Call failed; perhaps missing permission? - yield self.authorize_redirect() - return - self.finish("Posted a message!") - - .. testoutput:: - :hide: - - The given path is relative to ``self._FACEBOOK_BASE_URL``, - by default "https://graph.facebook.com". - - This method is a wrapper around `OAuth2Mixin.oauth2_request`; - the only difference is that this method takes a relative path, - while ``oauth2_request`` takes a complete url. - - .. versionchanged:: 3.1 - Added the ability to override ``self._FACEBOOK_BASE_URL``. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - url = self._FACEBOOK_BASE_URL + path - # Thanks to the _auth_return_future decorator, our "callback" - # argument is a Future, which we cannot pass as a callback to - # oauth2_request. Instead, have oauth2_request return a - # future and chain them together. - oauth_future = self.oauth2_request(url, access_token=access_token, - post_args=post_args, **args) - chain_future(oauth_future, callback) - - -def _oauth_signature(consumer_token, method, url, parameters={}, token=None): - """Calculates the HMAC-SHA1 OAuth signature for the given request. - - See http://oauth.net/core/1.0/#signing_process - """ - parts = urlparse.urlparse(url) - scheme, netloc, path = parts[:3] - normalized_url = scheme.lower() + "://" + netloc.lower() + path - - base_elems = [] - base_elems.append(method.upper()) - base_elems.append(normalized_url) - base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) - for k, v in sorted(parameters.items()))) - base_string = "&".join(_oauth_escape(e) for e in base_elems) - - key_elems = [escape.utf8(consumer_token["secret"])] - key_elems.append(escape.utf8(token["secret"] if token else "")) - key = b"&".join(key_elems) - - hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) - return binascii.b2a_base64(hash.digest())[:-1] - - -def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None): - """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request. - - See http://oauth.net/core/1.0a/#signing_process - """ - parts = urlparse.urlparse(url) - scheme, netloc, path = parts[:3] - normalized_url = scheme.lower() + "://" + netloc.lower() + path - - base_elems = [] - base_elems.append(method.upper()) - base_elems.append(normalized_url) - base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) - for k, v in sorted(parameters.items()))) - - base_string = "&".join(_oauth_escape(e) for e in base_elems) - key_elems = [escape.utf8(urllib_parse.quote(consumer_token["secret"], safe='~'))] - key_elems.append(escape.utf8(urllib_parse.quote(token["secret"], safe='~') if token else "")) - key = b"&".join(key_elems) - - hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) - return binascii.b2a_base64(hash.digest())[:-1] - - -def _oauth_escape(val): - if isinstance(val, unicode_type): - val = val.encode("utf-8") - return urllib_parse.quote(val, safe="~") - - -def _oauth_parse_response(body): - # I can't find an officially-defined encoding for oauth responses and - # have never seen anyone use non-ascii. Leave the response in a byte - # string for python 2, and use utf8 on python 3. - body = escape.native_str(body) - p = urlparse.parse_qs(body, keep_blank_values=False) - token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0]) - - # Add the extra parameters the Provider included to the token - special = ("oauth_token", "oauth_token_secret") - token.update((k, p[k][0]) for k in p if k not in special) - return token diff --git a/lib/tornado/autoreload.py b/lib/tornado/autoreload.py deleted file mode 100755 index 7d69474a..00000000 --- a/lib/tornado/autoreload.py +++ /dev/null @@ -1,356 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Automatically restart the server when a source file is modified. - -Most applications should not access this module directly. Instead, -pass the keyword argument ``autoreload=True`` to the -`tornado.web.Application` constructor (or ``debug=True``, which -enables this setting and several others). This will enable autoreload -mode as well as checking for changes to templates and static -resources. Note that restarting is a destructive operation and any -requests in progress will be aborted when the process restarts. (If -you want to disable autoreload while using other debug-mode features, -pass both ``debug=True`` and ``autoreload=False``). - -This module can also be used as a command-line wrapper around scripts -such as unit test runners. See the `main` method for details. - -The command-line wrapper and Application debug modes can be used together. -This combination is encouraged as the wrapper catches syntax errors and -other import-time failures, while debug mode catches changes once -the server has started. - -This module depends on `.IOLoop`, so it will not work in WSGI applications -and Google App Engine. It also will not work correctly when `.HTTPServer`'s -multi-process mode is used. - -Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) -because it re-executes Python using ``sys.executable`` and ``sys.argv``. -Additionally, modifying these variables will cause reloading to behave -incorrectly. - -""" - -from __future__ import absolute_import, division, print_function - -import os -import sys - -# sys.path handling -# ----------------- -# -# If a module is run with "python -m", the current directory (i.e. "") -# is automatically prepended to sys.path, but not if it is run as -# "path/to/file.py". The processing for "-m" rewrites the former to -# the latter, so subsequent executions won't have the same path as the -# original. -# -# Conversely, when run as path/to/file.py, the directory containing -# file.py gets added to the path, which can cause confusion as imports -# may become relative in spite of the future import. -# -# We address the former problem by reconstructing the original command -# line (Python >= 3.4) or by setting the $PYTHONPATH environment -# variable (Python < 3.4) before re-execution so the new process will -# see the correct path. We attempt to address the latter problem when -# tornado.autoreload is run as __main__. - -if __name__ == "__main__": - # This sys.path manipulation must come before our imports (as much - # as possible - if we introduced a tornado.sys or tornado.os - # module we'd be in trouble), or else our imports would become - # relative again despite the future import. - # - # There is a separate __main__ block at the end of the file to call main(). - if sys.path[0] == os.path.dirname(__file__): - del sys.path[0] - -import functools -import logging -import os -import pkgutil # type: ignore -import sys -import traceback -import types -import subprocess -import weakref - -from tornado import ioloop -from tornado.log import gen_log -from tornado import process -from tornado.util import exec_in - -try: - import signal -except ImportError: - signal = None - -# os.execv is broken on Windows and can't properly parse command line -# arguments and executable name if they contain whitespaces. subprocess -# fixes that behavior. -_has_execv = sys.platform != 'win32' - -_watched_files = set() -_reload_hooks = [] -_reload_attempted = False -_io_loops = weakref.WeakKeyDictionary() # type: ignore -_autoreload_is_main = False -_original_argv = None -_original_spec = None - - -def start(check_time=500): - """Begins watching source files for changes. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - """ - io_loop = ioloop.IOLoop.current() - if io_loop in _io_loops: - return - _io_loops[io_loop] = True - if len(_io_loops) > 1: - gen_log.warning("tornado.autoreload started more than once in the same process") - modify_times = {} - callback = functools.partial(_reload_on_update, modify_times) - scheduler = ioloop.PeriodicCallback(callback, check_time) - scheduler.start() - - -def wait(): - """Wait for a watched file to change, then restart the process. - - Intended to be used at the end of scripts like unit test runners, - to run the tests again after any source file changes (but see also - the command-line interface in `main`) - """ - io_loop = ioloop.IOLoop() - io_loop.add_callback(start) - io_loop.start() - - -def watch(filename): - """Add a file to the watch list. - - All imported modules are watched by default. - """ - _watched_files.add(filename) - - -def add_reload_hook(fn): - """Add a function to be called before reloading the process. - - Note that for open file and socket handles it is generally - preferable to set the ``FD_CLOEXEC`` flag (using `fcntl` or - ``tornado.platform.auto.set_close_exec``) instead - of using a reload hook to close them. - """ - _reload_hooks.append(fn) - - -def _reload_on_update(modify_times): - if _reload_attempted: - # We already tried to reload and it didn't work, so don't try again. - return - if process.task_id() is not None: - # We're in a child process created by fork_processes. If child - # processes restarted themselves, they'd all restart and then - # all call fork_processes again. - return - for module in list(sys.modules.values()): - # Some modules play games with sys.modules (e.g. email/__init__.py - # in the standard library), and occasionally this can cause strange - # failures in getattr. Just ignore anything that's not an ordinary - # module. - if not isinstance(module, types.ModuleType): - continue - path = getattr(module, "__file__", None) - if not path: - continue - if path.endswith(".pyc") or path.endswith(".pyo"): - path = path[:-1] - _check_file(modify_times, path) - for path in _watched_files: - _check_file(modify_times, path) - - -def _check_file(modify_times, path): - try: - modified = os.stat(path).st_mtime - except Exception: - return - if path not in modify_times: - modify_times[path] = modified - return - if modify_times[path] != modified: - gen_log.info("%s modified; restarting server", path) - _reload() - - -def _reload(): - global _reload_attempted - _reload_attempted = True - for fn in _reload_hooks: - fn() - if hasattr(signal, "setitimer"): - # Clear the alarm signal set by - # ioloop.set_blocking_log_threshold so it doesn't fire - # after the exec. - signal.setitimer(signal.ITIMER_REAL, 0, 0) - # sys.path fixes: see comments at top of file. If __main__.__spec__ - # exists, we were invoked with -m and the effective path is about to - # change on re-exec. Reconstruct the original command line to - # ensure that the new process sees the same path we did. If - # __spec__ is not available (Python < 3.4), check instead if - # sys.path[0] is an empty string and add the current directory to - # $PYTHONPATH. - if _autoreload_is_main: - spec = _original_spec - argv = _original_argv - else: - spec = getattr(sys.modules['__main__'], '__spec__', None) - argv = sys.argv - if spec: - argv = ['-m', spec.name] + argv[1:] - else: - path_prefix = '.' + os.pathsep - if (sys.path[0] == '' and - not os.environ.get("PYTHONPATH", "").startswith(path_prefix)): - os.environ["PYTHONPATH"] = (path_prefix + - os.environ.get("PYTHONPATH", "")) - if not _has_execv: - subprocess.Popen([sys.executable] + argv) - os._exit(0) - else: - try: - os.execv(sys.executable, [sys.executable] + argv) - except OSError: - # Mac OS X versions prior to 10.6 do not support execv in - # a process that contains multiple threads. Instead of - # re-executing in the current process, start a new one - # and cause the current process to exit. This isn't - # ideal since the new process is detached from the parent - # terminal and thus cannot easily be killed with ctrl-C, - # but it's better than not being able to autoreload at - # all. - # Unfortunately the errno returned in this case does not - # appear to be consistent, so we can't easily check for - # this error specifically. - os.spawnv(os.P_NOWAIT, sys.executable, [sys.executable] + argv) - # At this point the IOLoop has been closed and finally - # blocks will experience errors if we allow the stack to - # unwind, so just exit uncleanly. - os._exit(0) - - -_USAGE = """\ -Usage: - python -m tornado.autoreload -m module.to.run [args...] - python -m tornado.autoreload path/to/script.py [args...] -""" - - -def main(): - """Command-line wrapper to re-run a script whenever its source changes. - - Scripts may be specified by filename or module name:: - - python -m tornado.autoreload -m tornado.test.runtests - python -m tornado.autoreload tornado/test/runtests.py - - Running a script with this wrapper is similar to calling - `tornado.autoreload.wait` at the end of the script, but this wrapper - can catch import-time problems like syntax errors that would otherwise - prevent the script from reaching its call to `wait`. - """ - # Remember that we were launched with autoreload as main. - # The main module can be tricky; set the variables both in our globals - # (which may be __main__) and the real importable version. - import tornado.autoreload - global _autoreload_is_main - global _original_argv, _original_spec - tornado.autoreload._autoreload_is_main = _autoreload_is_main = True - original_argv = sys.argv - tornado.autoreload._original_argv = _original_argv = original_argv - original_spec = getattr(sys.modules['__main__'], '__spec__', None) - tornado.autoreload._original_spec = _original_spec = original_spec - sys.argv = sys.argv[:] - if len(sys.argv) >= 3 and sys.argv[1] == "-m": - mode = "module" - module = sys.argv[2] - del sys.argv[1:3] - elif len(sys.argv) >= 2: - mode = "script" - script = sys.argv[1] - sys.argv = sys.argv[1:] - else: - print(_USAGE, file=sys.stderr) - sys.exit(1) - - try: - if mode == "module": - import runpy - runpy.run_module(module, run_name="__main__", alter_sys=True) - elif mode == "script": - with open(script) as f: - # Execute the script in our namespace instead of creating - # a new one so that something that tries to import __main__ - # (e.g. the unittest module) will see names defined in the - # script instead of just those defined in this module. - global __file__ - __file__ = script - # If __package__ is defined, imports may be incorrectly - # interpreted as relative to this module. - global __package__ - del __package__ - exec_in(f.read(), globals(), globals()) - except SystemExit as e: - logging.basicConfig() - gen_log.info("Script exited with status %s", e.code) - except Exception as e: - logging.basicConfig() - gen_log.warning("Script exited with uncaught exception", exc_info=True) - # If an exception occurred at import time, the file with the error - # never made it into sys.modules and so we won't know to watch it. - # Just to make sure we've covered everything, walk the stack trace - # from the exception and watch every file. - for (filename, lineno, name, line) in traceback.extract_tb(sys.exc_info()[2]): - watch(filename) - if isinstance(e, SyntaxError): - # SyntaxErrors are special: their innermost stack frame is fake - # so extract_tb won't see it and we have to get the filename - # from the exception object. - watch(e.filename) - else: - logging.basicConfig() - gen_log.info("Script exited normally") - # restore sys.argv so subsequent executions will include autoreload - sys.argv = original_argv - - if mode == 'module': - # runpy did a fake import of the module as __main__, but now it's - # no longer in sys.modules. Figure out where it is and watch it. - loader = pkgutil.get_loader(module) - if loader is not None: - watch(loader.get_filename()) - - wait() - - -if __name__ == "__main__": - # See also the other __main__ block at the top of the file, which modifies - # sys.path before our imports - main() diff --git a/lib/tornado/concurrent.py b/lib/tornado/concurrent.py deleted file mode 100755 index f7e6bccc..00000000 --- a/lib/tornado/concurrent.py +++ /dev/null @@ -1,660 +0,0 @@ -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Utilities for working with ``Future`` objects. - -``Futures`` are a pattern for concurrent programming introduced in -Python 3.2 in the `concurrent.futures` package, and also adopted (in a -slightly different form) in Python 3.4's `asyncio` package. This -package defines a ``Future`` class that is an alias for `asyncio.Future` -when available, and a compatible implementation for older versions of -Python. It also includes some utility functions for interacting with -``Future`` objects. - -While this package is an important part of Tornado's internal -implementation, applications rarely need to interact with it -directly. -""" -from __future__ import absolute_import, division, print_function - -import functools -import platform -import textwrap -import traceback -import sys -import warnings - -from tornado.log import app_log -from tornado.stack_context import ExceptionStackContext, wrap -from tornado.util import raise_exc_info, ArgReplacer, is_finalizing - -try: - from concurrent import futures -except ImportError: - futures = None - -try: - import asyncio -except ImportError: - asyncio = None - -try: - import typing -except ImportError: - typing = None - - -# Can the garbage collector handle cycles that include __del__ methods? -# This is true in cpython beginning with version 3.4 (PEP 442). -_GC_CYCLE_FINALIZERS = (platform.python_implementation() == 'CPython' and - sys.version_info >= (3, 4)) - - -class ReturnValueIgnoredError(Exception): - pass - -# This class and associated code in the future object is derived -# from the Trollius project, a backport of asyncio to Python 2.x - 3.x - - -class _TracebackLogger(object): - """Helper to log a traceback upon destruction if not cleared. - - This solves a nasty problem with Futures and Tasks that have an - exception set: if nobody asks for the exception, the exception is - never logged. This violates the Zen of Python: 'Errors should - never pass silently. Unless explicitly silenced.' - - However, we don't want to log the exception as soon as - set_exception() is called: if the calling code is written - properly, it will get the exception and handle it properly. But - we *do* want to log it if result() or exception() was never called - -- otherwise developers waste a lot of time wondering why their - buggy code fails silently. - - An earlier attempt added a __del__() method to the Future class - itself, but this backfired because the presence of __del__() - prevents garbage collection from breaking cycles. A way out of - this catch-22 is to avoid having a __del__() method on the Future - class itself, but instead to have a reference to a helper object - with a __del__() method that logs the traceback, where we ensure - that the helper object doesn't participate in cycles, and only the - Future has a reference to it. - - The helper object is added when set_exception() is called. When - the Future is collected, and the helper is present, the helper - object is also collected, and its __del__() method will log the - traceback. When the Future's result() or exception() method is - called (and a helper object is present), it removes the the helper - object, after calling its clear() method to prevent it from - logging. - - One downside is that we do a fair amount of work to extract the - traceback from the exception, even when it is never logged. It - would seem cheaper to just store the exception object, but that - references the traceback, which references stack frames, which may - reference the Future, which references the _TracebackLogger, and - then the _TracebackLogger would be included in a cycle, which is - what we're trying to avoid! As an optimization, we don't - immediately format the exception; we only do the work when - activate() is called, which call is delayed until after all the - Future's callbacks have run. Since usually a Future has at least - one callback (typically set by 'yield From') and usually that - callback extracts the callback, thereby removing the need to - format the exception. - - PS. I don't claim credit for this solution. I first heard of it - in a discussion about closing files when they are collected. - """ - - __slots__ = ('exc_info', 'formatted_tb') - - def __init__(self, exc_info): - self.exc_info = exc_info - self.formatted_tb = None - - def activate(self): - exc_info = self.exc_info - if exc_info is not None: - self.exc_info = None - self.formatted_tb = traceback.format_exception(*exc_info) - - def clear(self): - self.exc_info = None - self.formatted_tb = None - - def __del__(self, is_finalizing=is_finalizing): - if not is_finalizing() and self.formatted_tb: - app_log.error('Future exception was never retrieved: %s', - ''.join(self.formatted_tb).rstrip()) - - -class Future(object): - """Placeholder for an asynchronous result. - - A ``Future`` encapsulates the result of an asynchronous - operation. In synchronous applications ``Futures`` are used - to wait for the result from a thread or process pool; in - Tornado they are normally used with `.IOLoop.add_future` or by - yielding them in a `.gen.coroutine`. - - `tornado.concurrent.Future` is an alias for `asyncio.Future` when - that package is available (Python 3.4+). Unlike - `concurrent.futures.Future`, the ``Futures`` used by Tornado and - `asyncio` are not thread-safe (and therefore faster for use with - single-threaded event loops). - - In addition to ``exception`` and ``set_exception``, Tornado's - ``Future`` implementation supports storing an ``exc_info`` triple - to support better tracebacks on Python 2. To set an ``exc_info`` - triple, use `future_set_exc_info`, and to retrieve one, call - `result()` (which will raise it). - - .. versionchanged:: 4.0 - `tornado.concurrent.Future` is always a thread-unsafe ``Future`` - with support for the ``exc_info`` methods. Previously it would - be an alias for the thread-safe `concurrent.futures.Future` - if that package was available and fall back to the thread-unsafe - implementation if it was not. - - .. versionchanged:: 4.1 - If a `.Future` contains an error but that error is never observed - (by calling ``result()``, ``exception()``, or ``exc_info()``), - a stack trace will be logged when the `.Future` is garbage collected. - This normally indicates an error in the application, but in cases - where it results in undesired logging it may be necessary to - suppress the logging by ensuring that the exception is observed: - ``f.add_done_callback(lambda f: f.exception())``. - - .. versionchanged:: 5.0 - - This class was previoiusly available under the name - ``TracebackFuture``. This name, which was deprecated since - version 4.0, has been removed. When `asyncio` is available - ``tornado.concurrent.Future`` is now an alias for - `asyncio.Future`. Like `asyncio.Future`, callbacks are now - always scheduled on the `.IOLoop` and are never run - synchronously. - - """ - def __init__(self): - self._done = False - self._result = None - self._exc_info = None - - self._log_traceback = False # Used for Python >= 3.4 - self._tb_logger = None # Used for Python <= 3.3 - - self._callbacks = [] - - # Implement the Python 3.5 Awaitable protocol if possible - # (we can't use return and yield together until py33). - if sys.version_info >= (3, 3): - exec(textwrap.dedent(""" - def __await__(self): - return (yield self) - """)) - else: - # Py2-compatible version for use with cython. - def __await__(self): - result = yield self - # StopIteration doesn't take args before py33, - # but Cython recognizes the args tuple. - e = StopIteration() - e.args = (result,) - raise e - - def cancel(self): - """Cancel the operation, if possible. - - Tornado ``Futures`` do not support cancellation, so this method always - returns False. - """ - return False - - def cancelled(self): - """Returns True if the operation has been cancelled. - - Tornado ``Futures`` do not support cancellation, so this method - always returns False. - """ - return False - - def running(self): - """Returns True if this operation is currently running.""" - return not self._done - - def done(self): - """Returns True if the future has finished running.""" - return self._done - - def _clear_tb_log(self): - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None - - def result(self, timeout=None): - """If the operation succeeded, return its result. If it failed, - re-raise its exception. - - This method takes a ``timeout`` argument for compatibility with - `concurrent.futures.Future` but it is an error to call it - before the `Future` is done, so the ``timeout`` is never used. - """ - self._clear_tb_log() - if self._result is not None: - return self._result - if self._exc_info is not None: - try: - raise_exc_info(self._exc_info) - finally: - self = None - self._check_done() - return self._result - - def exception(self, timeout=None): - """If the operation raised an exception, return the `Exception` - object. Otherwise returns None. - - This method takes a ``timeout`` argument for compatibility with - `concurrent.futures.Future` but it is an error to call it - before the `Future` is done, so the ``timeout`` is never used. - """ - self._clear_tb_log() - if self._exc_info is not None: - return self._exc_info[1] - else: - self._check_done() - return None - - def add_done_callback(self, fn): - """Attaches the given callback to the `Future`. - - It will be invoked with the `Future` as its argument when the Future - has finished running and its result is available. In Tornado - consider using `.IOLoop.add_future` instead of calling - `add_done_callback` directly. - """ - if self._done: - from tornado.ioloop import IOLoop - IOLoop.current().add_callback(fn, self) - else: - self._callbacks.append(fn) - - def set_result(self, result): - """Sets the result of a ``Future``. - - It is undefined to call any of the ``set`` methods more than once - on the same object. - """ - self._result = result - self._set_done() - - def set_exception(self, exception): - """Sets the exception of a ``Future.``""" - self.set_exc_info( - (exception.__class__, - exception, - getattr(exception, '__traceback__', None))) - - def exc_info(self): - """Returns a tuple in the same format as `sys.exc_info` or None. - - .. versionadded:: 4.0 - """ - self._clear_tb_log() - return self._exc_info - - def set_exc_info(self, exc_info): - """Sets the exception information of a ``Future.`` - - Preserves tracebacks on Python 2. - - .. versionadded:: 4.0 - """ - self._exc_info = exc_info - self._log_traceback = True - if not _GC_CYCLE_FINALIZERS: - self._tb_logger = _TracebackLogger(exc_info) - - try: - self._set_done() - finally: - # Activate the logger after all callbacks have had a - # chance to call result() or exception(). - if self._log_traceback and self._tb_logger is not None: - self._tb_logger.activate() - self._exc_info = exc_info - - def _check_done(self): - if not self._done: - raise Exception("DummyFuture does not support blocking for results") - - def _set_done(self): - self._done = True - if self._callbacks: - from tornado.ioloop import IOLoop - loop = IOLoop.current() - for cb in self._callbacks: - loop.add_callback(cb, self) - self._callbacks = None - - # On Python 3.3 or older, objects with a destructor part of a reference - # cycle are never destroyed. It's no longer the case on Python 3.4 thanks to - # the PEP 442. - if _GC_CYCLE_FINALIZERS: - def __del__(self, is_finalizing=is_finalizing): - if is_finalizing() or not self._log_traceback: - # set_exception() was not called, or result() or exception() - # has consumed the exception - return - - tb = traceback.format_exception(*self._exc_info) - - app_log.error('Future %r exception was never retrieved: %s', - self, ''.join(tb).rstrip()) - - -if asyncio is not None: - Future = asyncio.Future # noqa - -if futures is None: - FUTURES = Future # type: typing.Union[type, typing.Tuple[type, ...]] -else: - FUTURES = (futures.Future, Future) - - -def is_future(x): - return isinstance(x, FUTURES) - - -class DummyExecutor(object): - def submit(self, fn, *args, **kwargs): - future = Future() - try: - future_set_result_unless_cancelled(future, fn(*args, **kwargs)) - except Exception: - future_set_exc_info(future, sys.exc_info()) - return future - - def shutdown(self, wait=True): - pass - - -dummy_executor = DummyExecutor() - - -def run_on_executor(*args, **kwargs): - """Decorator to run a synchronous method asynchronously on an executor. - - The decorated method may be called with a ``callback`` keyword - argument and returns a future. - - The executor to be used is determined by the ``executor`` - attributes of ``self``. To use a different attribute name, pass a - keyword argument to the decorator:: - - @run_on_executor(executor='_thread_pool') - def foo(self): - pass - - This decorator should not be confused with the similarly-named - `.IOLoop.run_in_executor`. In general, using ``run_in_executor`` - when *calling* a blocking method is recommended instead of using - this decorator when *defining* a method. If compatibility with older - versions of Tornado is required, consider defining an executor - and using ``executor.submit()`` at the call site. - - .. versionchanged:: 4.2 - Added keyword arguments to use alternative attributes. - - .. versionchanged:: 5.0 - Always uses the current IOLoop instead of ``self.io_loop``. - - .. versionchanged:: 5.1 - Returns a `.Future` compatible with ``await`` instead of a - `concurrent.futures.Future`. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in - 6.0. The decorator itself is discouraged in new code but will - not be removed in 6.0. - """ - def run_on_executor_decorator(fn): - executor = kwargs.get("executor", "executor") - - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - callback = kwargs.pop("callback", None) - async_future = Future() - conc_future = getattr(self, executor).submit(fn, self, *args, **kwargs) - chain_future(conc_future, async_future) - if callback: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - from tornado.ioloop import IOLoop - IOLoop.current().add_future( - async_future, lambda future: callback(future.result())) - return async_future - return wrapper - if args and kwargs: - raise ValueError("cannot combine positional and keyword args") - if len(args) == 1: - return run_on_executor_decorator(args[0]) - elif len(args) != 0: - raise ValueError("expected 1 argument, got %d", len(args)) - return run_on_executor_decorator - - -_NO_RESULT = object() - - -def return_future(f): - """Decorator to make a function that returns via callback return a - `Future`. - - This decorator was provided to ease the transition from - callback-oriented code to coroutines. It is not recommended for - new code. - - The wrapped function should take a ``callback`` keyword argument - and invoke it with one argument when it has finished. To signal failure, - the function can simply raise an exception (which will be - captured by the `.StackContext` and passed along to the ``Future``). - - From the caller's perspective, the callback argument is optional. - If one is given, it will be invoked when the function is complete - with ``Future.result()`` as an argument. If the function fails, the - callback will not be run and an exception will be raised into the - surrounding `.StackContext`. - - If no callback is given, the caller should use the ``Future`` to - wait for the function to complete (perhaps by yielding it in a - coroutine, or passing it to `.IOLoop.add_future`). - - Usage: - - .. testcode:: - - @return_future - def future_func(arg1, arg2, callback): - # Do stuff (possibly asynchronous) - callback(result) - - async def caller(): - await future_func(arg1, arg2) - - .. - - Note that ``@return_future`` and ``@gen.engine`` can be applied to the - same function, provided ``@return_future`` appears first. However, - consider using ``@gen.coroutine`` instead of this combination. - - .. versionchanged:: 5.1 - - Now raises a `.DeprecationWarning` if a callback argument is passed to - the decorated function and deprecation warnings are enabled. - - .. deprecated:: 5.1 - - This decorator will be removed in Tornado 6.0. New code should - use coroutines directly instead of wrapping callback-based code - with this decorator. Interactions with non-Tornado - callback-based code should be managed explicitly to avoid - relying on the `.ExceptionStackContext` built into this - decorator. - """ - warnings.warn("@return_future is deprecated, use coroutines instead", - DeprecationWarning) - return _non_deprecated_return_future(f, warn=True) - - -def _non_deprecated_return_future(f, warn=False): - # Allow auth.py to use this decorator without triggering - # deprecation warnings. This will go away once auth.py has removed - # its legacy interfaces in 6.0. - replacer = ArgReplacer(f, 'callback') - - @functools.wraps(f) - def wrapper(*args, **kwargs): - future = Future() - callback, args, kwargs = replacer.replace( - lambda value=_NO_RESULT: future_set_result_unless_cancelled(future, value), - args, kwargs) - - def handle_error(typ, value, tb): - future_set_exc_info(future, (typ, value, tb)) - return True - exc_info = None - esc = ExceptionStackContext(handle_error, delay_warning=True) - with esc: - if not warn: - # HACK: In non-deprecated mode (only used in auth.py), - # suppress the warning entirely. Since this is added - # in a 5.1 patch release and already removed in 6.0 - # I'm prioritizing a minimial change instead of a - # clean solution. - esc.delay_warning = False - try: - result = f(*args, **kwargs) - if result is not None: - raise ReturnValueIgnoredError( - "@return_future should not be used with functions " - "that return values") - except: - exc_info = sys.exc_info() - raise - if exc_info is not None: - # If the initial synchronous part of f() raised an exception, - # go ahead and raise it to the caller directly without waiting - # for them to inspect the Future. - future.result() - - # If the caller passed in a callback, schedule it to be called - # when the future resolves. It is important that this happens - # just before we return the future, or else we risk confusing - # stack contexts with multiple exceptions (one here with the - # immediate exception, and again when the future resolves and - # the callback triggers its exception by calling future.result()). - if callback is not None: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - - def run_callback(future): - result = future.result() - if result is _NO_RESULT: - callback() - else: - callback(future.result()) - future_add_done_callback(future, wrap(run_callback)) - return future - return wrapper - - -def chain_future(a, b): - """Chain two futures together so that when one completes, so does the other. - - The result (success or failure) of ``a`` will be copied to ``b``, unless - ``b`` has already been completed or cancelled by the time ``a`` finishes. - - .. versionchanged:: 5.0 - - Now accepts both Tornado/asyncio `Future` objects and - `concurrent.futures.Future`. - - """ - def copy(future): - assert future is a - if b.done(): - return - if (hasattr(a, 'exc_info') and - a.exc_info() is not None): - future_set_exc_info(b, a.exc_info()) - elif a.exception() is not None: - b.set_exception(a.exception()) - else: - b.set_result(a.result()) - if isinstance(a, Future): - future_add_done_callback(a, copy) - else: - # concurrent.futures.Future - from tornado.ioloop import IOLoop - IOLoop.current().add_future(a, copy) - - -def future_set_result_unless_cancelled(future, value): - """Set the given ``value`` as the `Future`'s result, if not cancelled. - - Avoids asyncio.InvalidStateError when calling set_result() on - a cancelled `asyncio.Future`. - - .. versionadded:: 5.0 - """ - if not future.cancelled(): - future.set_result(value) - - -def future_set_exc_info(future, exc_info): - """Set the given ``exc_info`` as the `Future`'s exception. - - Understands both `asyncio.Future` and Tornado's extensions to - enable better tracebacks on Python 2. - - .. versionadded:: 5.0 - """ - if hasattr(future, 'set_exc_info'): - # Tornado's Future - future.set_exc_info(exc_info) - else: - # asyncio.Future - future.set_exception(exc_info[1]) - - -def future_add_done_callback(future, callback): - """Arrange to call ``callback`` when ``future`` is complete. - - ``callback`` is invoked with one argument, the ``future``. - - If ``future`` is already done, ``callback`` is invoked immediately. - This may differ from the behavior of ``Future.add_done_callback``, - which makes no such guarantee. - - .. versionadded:: 5.0 - """ - if future.done(): - callback(future) - else: - future.add_done_callback(callback) diff --git a/lib/tornado/curl_httpclient.py b/lib/tornado/curl_httpclient.py deleted file mode 100755 index 7f5cb105..00000000 --- a/lib/tornado/curl_httpclient.py +++ /dev/null @@ -1,514 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Non-blocking HTTP client implementation using pycurl.""" - -from __future__ import absolute_import, division, print_function - -import collections -import functools -import logging -import pycurl # type: ignore -import threading -import time -from io import BytesIO - -from tornado import httputil -from tornado import ioloop -from tornado import stack_context - -from tornado.escape import utf8, native_str -from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main - -curl_log = logging.getLogger('tornado.curl_httpclient') - - -class CurlAsyncHTTPClient(AsyncHTTPClient): - def initialize(self, max_clients=10, defaults=None): - super(CurlAsyncHTTPClient, self).initialize(defaults=defaults) - self._multi = pycurl.CurlMulti() - self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) - self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) - self._curls = [self._curl_create() for i in range(max_clients)] - self._free_list = self._curls[:] - self._requests = collections.deque() - self._fds = {} - self._timeout = None - - # libcurl has bugs that sometimes cause it to not report all - # relevant file descriptors and timeouts to TIMERFUNCTION/ - # SOCKETFUNCTION. Mitigate the effects of such bugs by - # forcing a periodic scan of all active requests. - self._force_timeout_callback = ioloop.PeriodicCallback( - self._handle_force_timeout, 1000) - self._force_timeout_callback.start() - - # Work around a bug in libcurl 7.29.0: Some fields in the curl - # multi object are initialized lazily, and its destructor will - # segfault if it is destroyed without having been used. Add - # and remove a dummy handle to make sure everything is - # initialized. - dummy_curl_handle = pycurl.Curl() - self._multi.add_handle(dummy_curl_handle) - self._multi.remove_handle(dummy_curl_handle) - - def close(self): - self._force_timeout_callback.stop() - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - for curl in self._curls: - curl.close() - self._multi.close() - super(CurlAsyncHTTPClient, self).close() - - # Set below properties to None to reduce the reference count of current - # instance, because those properties hold some methods of current - # instance that will case circular reference. - self._force_timeout_callback = None - self._multi = None - - def fetch_impl(self, request, callback): - self._requests.append((request, callback, self.io_loop.time())) - self._process_queue() - self._set_timeout(0) - - def _handle_socket(self, event, fd, multi, data): - """Called by libcurl when it wants to change the file descriptors - it cares about. - """ - event_map = { - pycurl.POLL_NONE: ioloop.IOLoop.NONE, - pycurl.POLL_IN: ioloop.IOLoop.READ, - pycurl.POLL_OUT: ioloop.IOLoop.WRITE, - pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE - } - if event == pycurl.POLL_REMOVE: - if fd in self._fds: - self.io_loop.remove_handler(fd) - del self._fds[fd] - else: - ioloop_event = event_map[event] - # libcurl sometimes closes a socket and then opens a new - # one using the same FD without giving us a POLL_NONE in - # between. This is a problem with the epoll IOLoop, - # because the kernel can tell when a socket is closed and - # removes it from the epoll automatically, causing future - # update_handler calls to fail. Since we can't tell when - # this has happened, always use remove and re-add - # instead of update. - if fd in self._fds: - self.io_loop.remove_handler(fd) - self.io_loop.add_handler(fd, self._handle_events, - ioloop_event) - self._fds[fd] = ioloop_event - - def _set_timeout(self, msecs): - """Called by libcurl to schedule a timeout.""" - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = self.io_loop.add_timeout( - self.io_loop.time() + msecs / 1000.0, self._handle_timeout) - - def _handle_events(self, fd, events): - """Called by IOLoop when there is activity on one of our - file descriptors. - """ - action = 0 - if events & ioloop.IOLoop.READ: - action |= pycurl.CSELECT_IN - if events & ioloop.IOLoop.WRITE: - action |= pycurl.CSELECT_OUT - while True: - try: - ret, num_handles = self._multi.socket_action(fd, action) - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() - - def _handle_timeout(self): - """Called by IOLoop when the requested timeout has passed.""" - with stack_context.NullContext(): - self._timeout = None - while True: - try: - ret, num_handles = self._multi.socket_action( - pycurl.SOCKET_TIMEOUT, 0) - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() - - # In theory, we shouldn't have to do this because curl will - # call _set_timeout whenever the timeout changes. However, - # sometimes after _handle_timeout we will need to reschedule - # immediately even though nothing has changed from curl's - # perspective. This is because when socket_action is - # called with SOCKET_TIMEOUT, libcurl decides internally which - # timeouts need to be processed by using a monotonic clock - # (where available) while tornado uses python's time.time() - # to decide when timeouts have occurred. When those clocks - # disagree on elapsed time (as they will whenever there is an - # NTP adjustment), tornado might call _handle_timeout before - # libcurl is ready. After each timeout, resync the scheduled - # timeout with libcurl's current state. - new_timeout = self._multi.timeout() - if new_timeout >= 0: - self._set_timeout(new_timeout) - - def _handle_force_timeout(self): - """Called by IOLoop periodically to ask libcurl to process any - events it may have forgotten about. - """ - with stack_context.NullContext(): - while True: - try: - ret, num_handles = self._multi.socket_all() - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() - - def _finish_pending_requests(self): - """Process any requests that were completed by the last - call to multi.socket_action. - """ - while True: - num_q, ok_list, err_list = self._multi.info_read() - for curl in ok_list: - self._finish(curl) - for curl, errnum, errmsg in err_list: - self._finish(curl, errnum, errmsg) - if num_q == 0: - break - self._process_queue() - - def _process_queue(self): - with stack_context.NullContext(): - while True: - started = 0 - while self._free_list and self._requests: - started += 1 - curl = self._free_list.pop() - (request, callback, queue_start_time) = self._requests.popleft() - curl.info = { - "headers": httputil.HTTPHeaders(), - "buffer": BytesIO(), - "request": request, - "callback": callback, - "queue_start_time": queue_start_time, - "curl_start_time": time.time(), - "curl_start_ioloop_time": self.io_loop.current().time(), - } - try: - self._curl_setup_request( - curl, request, curl.info["buffer"], - curl.info["headers"]) - except Exception as e: - # If there was an error in setup, pass it on - # to the callback. Note that allowing the - # error to escape here will appear to work - # most of the time since we are still in the - # caller's original stack frame, but when - # _process_queue() is called from - # _finish_pending_requests the exceptions have - # nowhere to go. - self._free_list.append(curl) - callback(HTTPResponse( - request=request, - code=599, - error=e)) - else: - self._multi.add_handle(curl) - - if not started: - break - - def _finish(self, curl, curl_error=None, curl_message=None): - info = curl.info - curl.info = None - self._multi.remove_handle(curl) - self._free_list.append(curl) - buffer = info["buffer"] - if curl_error: - error = CurlError(curl_error, curl_message) - code = error.code - effective_url = None - buffer.close() - buffer = None - else: - error = None - code = curl.getinfo(pycurl.HTTP_CODE) - effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) - buffer.seek(0) - # the various curl timings are documented at - # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html - time_info = dict( - queue=info["curl_start_ioloop_time"] - info["queue_start_time"], - namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), - connect=curl.getinfo(pycurl.CONNECT_TIME), - appconnect=curl.getinfo(pycurl.APPCONNECT_TIME), - pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), - starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), - total=curl.getinfo(pycurl.TOTAL_TIME), - redirect=curl.getinfo(pycurl.REDIRECT_TIME), - ) - try: - info["callback"](HTTPResponse( - request=info["request"], code=code, headers=info["headers"], - buffer=buffer, effective_url=effective_url, error=error, - reason=info['headers'].get("X-Http-Reason", None), - request_time=self.io_loop.time() - info["curl_start_ioloop_time"], - start_time=info["curl_start_time"], - time_info=time_info)) - except Exception: - self.handle_callback_exception(info["callback"]) - - def handle_callback_exception(self, callback): - self.io_loop.handle_callback_exception(callback) - - def _curl_create(self): - curl = pycurl.Curl() - if curl_log.isEnabledFor(logging.DEBUG): - curl.setopt(pycurl.VERBOSE, 1) - curl.setopt(pycurl.DEBUGFUNCTION, self._curl_debug) - if hasattr(pycurl, 'PROTOCOLS'): # PROTOCOLS first appeared in pycurl 7.19.5 (2014-07-12) - curl.setopt(pycurl.PROTOCOLS, pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS) - curl.setopt(pycurl.REDIR_PROTOCOLS, pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS) - return curl - - def _curl_setup_request(self, curl, request, buffer, headers): - curl.setopt(pycurl.URL, native_str(request.url)) - - # libcurl's magic "Expect: 100-continue" behavior causes delays - # with servers that don't support it (which include, among others, - # Google's OpenID endpoint). Additionally, this behavior has - # a bug in conjunction with the curl_multi_socket_action API - # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), - # which increases the delays. It's more trouble than it's worth, - # so just turn off the feature (yes, setting Expect: to an empty - # value is the official way to disable this) - if "Expect" not in request.headers: - request.headers["Expect"] = "" - - # libcurl adds Pragma: no-cache by default; disable that too - if "Pragma" not in request.headers: - request.headers["Pragma"] = "" - - curl.setopt(pycurl.HTTPHEADER, - ["%s: %s" % (native_str(k), native_str(v)) - for k, v in request.headers.get_all()]) - - curl.setopt(pycurl.HEADERFUNCTION, - functools.partial(self._curl_header_callback, - headers, request.header_callback)) - if request.streaming_callback: - def write_function(chunk): - self.io_loop.add_callback(request.streaming_callback, chunk) - else: - write_function = buffer.write - curl.setopt(pycurl.WRITEFUNCTION, write_function) - curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) - curl.setopt(pycurl.MAXREDIRS, request.max_redirects) - curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) - curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) - if request.user_agent: - curl.setopt(pycurl.USERAGENT, native_str(request.user_agent)) - else: - curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") - if request.network_interface: - curl.setopt(pycurl.INTERFACE, request.network_interface) - if request.decompress_response: - curl.setopt(pycurl.ENCODING, "gzip,deflate") - else: - curl.setopt(pycurl.ENCODING, "none") - if request.proxy_host and request.proxy_port: - curl.setopt(pycurl.PROXY, request.proxy_host) - curl.setopt(pycurl.PROXYPORT, request.proxy_port) - if request.proxy_username: - credentials = httputil.encode_username_password(request.proxy_username, - request.proxy_password) - curl.setopt(pycurl.PROXYUSERPWD, credentials) - - if (request.proxy_auth_mode is None or - request.proxy_auth_mode == "basic"): - curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC) - elif request.proxy_auth_mode == "digest": - curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_DIGEST) - else: - raise ValueError( - "Unsupported proxy_auth_mode %s" % request.proxy_auth_mode) - else: - curl.setopt(pycurl.PROXY, '') - curl.unsetopt(pycurl.PROXYUSERPWD) - if request.validate_cert: - curl.setopt(pycurl.SSL_VERIFYPEER, 1) - curl.setopt(pycurl.SSL_VERIFYHOST, 2) - else: - curl.setopt(pycurl.SSL_VERIFYPEER, 0) - curl.setopt(pycurl.SSL_VERIFYHOST, 0) - if request.ca_certs is not None: - curl.setopt(pycurl.CAINFO, request.ca_certs) - else: - # There is no way to restore pycurl.CAINFO to its default value - # (Using unsetopt makes it reject all certificates). - # I don't see any way to read the default value from python so it - # can be restored later. We'll have to just leave CAINFO untouched - # if no ca_certs file was specified, and require that if any - # request uses a custom ca_certs file, they all must. - pass - - if request.allow_ipv6 is False: - # Curl behaves reasonably when DNS resolution gives an ipv6 address - # that we can't reach, so allow ipv6 unless the user asks to disable. - curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) - else: - curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) - - # Set the request method through curl's irritating interface which makes - # up names for almost every single method - curl_options = { - "GET": pycurl.HTTPGET, - "POST": pycurl.POST, - "PUT": pycurl.UPLOAD, - "HEAD": pycurl.NOBODY, - } - custom_methods = set(["DELETE", "OPTIONS", "PATCH"]) - for o in curl_options.values(): - curl.setopt(o, False) - if request.method in curl_options: - curl.unsetopt(pycurl.CUSTOMREQUEST) - curl.setopt(curl_options[request.method], True) - elif request.allow_nonstandard_methods or request.method in custom_methods: - curl.setopt(pycurl.CUSTOMREQUEST, request.method) - else: - raise KeyError('unknown method ' + request.method) - - body_expected = request.method in ("POST", "PATCH", "PUT") - body_present = request.body is not None - if not request.allow_nonstandard_methods: - # Some HTTP methods nearly always have bodies while others - # almost never do. Fail in this case unless the user has - # opted out of sanity checks with allow_nonstandard_methods. - if ((body_expected and not body_present) or - (body_present and not body_expected)): - raise ValueError( - 'Body must %sbe None for method %s (unless ' - 'allow_nonstandard_methods is true)' % - ('not ' if body_expected else '', request.method)) - - if body_expected or body_present: - if request.method == "GET": - # Even with `allow_nonstandard_methods` we disallow - # GET with a body (because libcurl doesn't allow it - # unless we use CUSTOMREQUEST). While the spec doesn't - # forbid clients from sending a body, it arguably - # disallows the server from doing anything with them. - raise ValueError('Body must be None for GET request') - request_buffer = BytesIO(utf8(request.body or '')) - - def ioctl(cmd): - if cmd == curl.IOCMD_RESTARTREAD: - request_buffer.seek(0) - curl.setopt(pycurl.READFUNCTION, request_buffer.read) - curl.setopt(pycurl.IOCTLFUNCTION, ioctl) - if request.method == "POST": - curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or '')) - else: - curl.setopt(pycurl.UPLOAD, True) - curl.setopt(pycurl.INFILESIZE, len(request.body or '')) - - if request.auth_username is not None: - if request.auth_mode is None or request.auth_mode == "basic": - curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) - elif request.auth_mode == "digest": - curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) - else: - raise ValueError("Unsupported auth_mode %s" % request.auth_mode) - - userpwd = httputil.encode_username_password(request.auth_username, - request.auth_password) - curl.setopt(pycurl.USERPWD, userpwd) - curl_log.debug("%s %s (username: %r)", request.method, request.url, - request.auth_username) - else: - curl.unsetopt(pycurl.USERPWD) - curl_log.debug("%s %s", request.method, request.url) - - if request.client_cert is not None: - curl.setopt(pycurl.SSLCERT, request.client_cert) - - if request.client_key is not None: - curl.setopt(pycurl.SSLKEY, request.client_key) - - if request.ssl_options is not None: - raise ValueError("ssl_options not supported in curl_httpclient") - - if threading.activeCount() > 1: - # libcurl/pycurl is not thread-safe by default. When multiple threads - # are used, signals should be disabled. This has the side effect - # of disabling DNS timeouts in some environments (when libcurl is - # not linked against ares), so we don't do it when there is only one - # thread. Applications that use many short-lived threads may need - # to set NOSIGNAL manually in a prepare_curl_callback since - # there may not be any other threads running at the time we call - # threading.activeCount. - curl.setopt(pycurl.NOSIGNAL, 1) - if request.prepare_curl_callback is not None: - request.prepare_curl_callback(curl) - - def _curl_header_callback(self, headers, header_callback, header_line): - header_line = native_str(header_line.decode('latin1')) - if header_callback is not None: - self.io_loop.add_callback(header_callback, header_line) - # header_line as returned by curl includes the end-of-line characters. - # whitespace at the start should be preserved to allow multi-line headers - header_line = header_line.rstrip() - if header_line.startswith("HTTP/"): - headers.clear() - try: - (__, __, reason) = httputil.parse_response_start_line(header_line) - header_line = "X-Http-Reason: %s" % reason - except httputil.HTTPInputError: - return - if not header_line: - return - headers.parse_line(header_line) - - def _curl_debug(self, debug_type, debug_msg): - debug_types = ('I', '<', '>', '<', '>') - if debug_type == 0: - debug_msg = native_str(debug_msg) - curl_log.debug('%s', debug_msg.strip()) - elif debug_type in (1, 2): - debug_msg = native_str(debug_msg) - for line in debug_msg.splitlines(): - curl_log.debug('%s %s', debug_types[debug_type], line) - elif debug_type == 4: - curl_log.debug('%s %r', debug_types[debug_type], debug_msg) - - -class CurlError(HTTPError): - def __init__(self, errno, message): - HTTPError.__init__(self, 599, message) - self.errno = errno - - -if __name__ == "__main__": - AsyncHTTPClient.configure(CurlAsyncHTTPClient) - main() diff --git a/lib/tornado/escape.py b/lib/tornado/escape.py deleted file mode 100755 index a79ece66..00000000 --- a/lib/tornado/escape.py +++ /dev/null @@ -1,399 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Escaping/unescaping methods for HTML, JSON, URLs, and others. - -Also includes a few other miscellaneous string manipulation functions that -have crept in over time. -""" - -from __future__ import absolute_import, division, print_function - -import json -import re - -from tornado.util import PY3, unicode_type, basestring_type - -if PY3: - from urllib.parse import parse_qs as _parse_qs - import html.entities as htmlentitydefs - import urllib.parse as urllib_parse - unichr = chr -else: - from urlparse import parse_qs as _parse_qs - import htmlentitydefs - import urllib as urllib_parse - -try: - import typing # noqa -except ImportError: - pass - - -_XHTML_ESCAPE_RE = re.compile('[&<>"\']') -_XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"', - '\'': '''} - - -def xhtml_escape(value): - """Escapes a string so it is valid within HTML or XML. - - Escapes the characters ``<``, ``>``, ``"``, ``'``, and ``&``. - When used in attribute values the escaped strings must be enclosed - in quotes. - - .. versionchanged:: 3.2 - - Added the single quote to the list of escaped characters. - """ - return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], - to_basestring(value)) - - -def xhtml_unescape(value): - """Un-escapes an XML-escaped string.""" - return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) - - -# The fact that json_encode wraps json.dumps is an implementation detail. -# Please see https://github.com/tornadoweb/tornado/pull/706 -# before sending a pull request that adds **kwargs to this function. -def json_encode(value): - """JSON-encodes the given Python object.""" - # JSON permits but does not require forward slashes to be escaped. - # This is useful when json data is emitted in a <script> tag - # in HTML, as it prevents </script> tags from prematurely terminating - # the javascript. Some json libraries do this escaping by default, - # although python's standard library does not, so we do it here. - # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped - return json.dumps(value).replace("</", "<\\/") - - -def json_decode(value): - """Returns Python objects for the given JSON string.""" - return json.loads(to_basestring(value)) - - -def squeeze(value): - """Replace all sequences of whitespace chars with a single space.""" - return re.sub(r"[\x00-\x20]+", " ", value).strip() - - -def url_escape(value, plus=True): - """Returns a URL-encoded version of the given value. - - If ``plus`` is true (the default), spaces will be represented - as "+" instead of "%20". This is appropriate for query strings - but not for the path component of a URL. Note that this default - is the reverse of Python's urllib module. - - .. versionadded:: 3.1 - The ``plus`` argument - """ - quote = urllib_parse.quote_plus if plus else urllib_parse.quote - return quote(utf8(value)) - - -# python 3 changed things around enough that we need two separate -# implementations of url_unescape. We also need our own implementation -# of parse_qs since python 3's version insists on decoding everything. -if not PY3: - def url_unescape(value, encoding='utf-8', plus=True): - """Decodes the given value from a URL. - - The argument may be either a byte or unicode string. - - If encoding is None, the result will be a byte string. Otherwise, - the result is a unicode string in the specified encoding. - - If ``plus`` is true (the default), plus signs will be interpreted - as spaces (literal plus signs must be represented as "%2B"). This - is appropriate for query strings and form-encoded values but not - for the path component of a URL. Note that this default is the - reverse of Python's urllib module. - - .. versionadded:: 3.1 - The ``plus`` argument - """ - unquote = (urllib_parse.unquote_plus if plus else urllib_parse.unquote) - if encoding is None: - return unquote(utf8(value)) - else: - return unicode_type(unquote(utf8(value)), encoding) - - parse_qs_bytes = _parse_qs -else: - def url_unescape(value, encoding='utf-8', plus=True): - """Decodes the given value from a URL. - - The argument may be either a byte or unicode string. - - If encoding is None, the result will be a byte string. Otherwise, - the result is a unicode string in the specified encoding. - - If ``plus`` is true (the default), plus signs will be interpreted - as spaces (literal plus signs must be represented as "%2B"). This - is appropriate for query strings and form-encoded values but not - for the path component of a URL. Note that this default is the - reverse of Python's urllib module. - - .. versionadded:: 3.1 - The ``plus`` argument - """ - if encoding is None: - if plus: - # unquote_to_bytes doesn't have a _plus variant - value = to_basestring(value).replace('+', ' ') - return urllib_parse.unquote_to_bytes(value) - else: - unquote = (urllib_parse.unquote_plus if plus - else urllib_parse.unquote) - return unquote(to_basestring(value), encoding=encoding) - - def parse_qs_bytes(qs, keep_blank_values=False, strict_parsing=False): - """Parses a query string like urlparse.parse_qs, but returns the - values as byte strings. - - Keys still become type str (interpreted as latin1 in python3!) - because it's too painful to keep them as byte strings in - python3 and in practice they're nearly always ascii anyway. - """ - # This is gross, but python3 doesn't give us another way. - # Latin1 is the universal donor of character encodings. - result = _parse_qs(qs, keep_blank_values, strict_parsing, - encoding='latin1', errors='strict') - encoded = {} - for k, v in result.items(): - encoded[k] = [i.encode('latin1') for i in v] - return encoded - - -_UTF8_TYPES = (bytes, type(None)) - - -def utf8(value): - # type: (typing.Union[bytes,unicode_type,None])->typing.Union[bytes,None] - """Converts a string argument to a byte string. - - If the argument is already a byte string or None, it is returned unchanged. - Otherwise it must be a unicode string and is encoded as utf8. - """ - if isinstance(value, _UTF8_TYPES): - return value - if not isinstance(value, unicode_type): - raise TypeError( - "Expected bytes, unicode, or None; got %r" % type(value) - ) - return value.encode("utf-8") - - -_TO_UNICODE_TYPES = (unicode_type, type(None)) - - -def to_unicode(value): - """Converts a string argument to a unicode string. - - If the argument is already a unicode string or None, it is returned - unchanged. Otherwise it must be a byte string and is decoded as utf8. - """ - if isinstance(value, _TO_UNICODE_TYPES): - return value - if not isinstance(value, bytes): - raise TypeError( - "Expected bytes, unicode, or None; got %r" % type(value) - ) - return value.decode("utf-8") - - -# to_unicode was previously named _unicode not because it was private, -# but to avoid conflicts with the built-in unicode() function/type -_unicode = to_unicode - -# When dealing with the standard library across python 2 and 3 it is -# sometimes useful to have a direct conversion to the native string type -if str is unicode_type: - native_str = to_unicode -else: - native_str = utf8 - -_BASESTRING_TYPES = (basestring_type, type(None)) - - -def to_basestring(value): - """Converts a string argument to a subclass of basestring. - - In python2, byte and unicode strings are mostly interchangeable, - so functions that deal with a user-supplied argument in combination - with ascii string constants can use either and should return the type - the user supplied. In python3, the two types are not interchangeable, - so this method is needed to convert byte strings to unicode. - """ - if isinstance(value, _BASESTRING_TYPES): - return value - if not isinstance(value, bytes): - raise TypeError( - "Expected bytes, unicode, or None; got %r" % type(value) - ) - return value.decode("utf-8") - - -def recursive_unicode(obj): - """Walks a simple data structure, converting byte strings to unicode. - - Supports lists, tuples, and dictionaries. - """ - if isinstance(obj, dict): - return dict((recursive_unicode(k), recursive_unicode(v)) for (k, v) in obj.items()) - elif isinstance(obj, list): - return list(recursive_unicode(i) for i in obj) - elif isinstance(obj, tuple): - return tuple(recursive_unicode(i) for i in obj) - elif isinstance(obj, bytes): - return to_unicode(obj) - else: - return obj - - -# I originally used the regex from -# http://daringfireball.net/2010/07/improved_regex_for_matching_urls -# but it gets all exponential on certain patterns (such as too many trailing -# dots), causing the regex matcher to never return. -# This regex should avoid those problems. -# Use to_unicode instead of tornado.util.u - we don't want backslashes getting -# processed as escapes. -_URL_RE = re.compile(to_unicode( - r"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""" # noqa: E501 -)) - - -def linkify(text, shorten=False, extra_params="", - require_protocol=False, permitted_protocols=["http", "https"]): - """Converts plain text into HTML with links. - - For example: ``linkify("Hello http://tornadoweb.org!")`` would return - ``Hello <a href="http://tornadoweb.org">http://tornadoweb.org</a>!`` - - Parameters: - - * ``shorten``: Long urls will be shortened for display. - - * ``extra_params``: Extra text to include in the link tag, or a callable - taking the link as an argument and returning the extra text - e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``, - or:: - - def extra_params_cb(url): - if url.startswith("http://example.com"): - return 'class="internal"' - else: - return 'class="external" rel="nofollow"' - linkify(text, extra_params=extra_params_cb) - - * ``require_protocol``: Only linkify urls which include a protocol. If - this is False, urls such as www.facebook.com will also be linkified. - - * ``permitted_protocols``: List (or set) of protocols which should be - linkified, e.g. ``linkify(text, permitted_protocols=["http", "ftp", - "mailto"])``. It is very unsafe to include protocols such as - ``javascript``. - """ - if extra_params and not callable(extra_params): - extra_params = " " + extra_params.strip() - - def make_link(m): - url = m.group(1) - proto = m.group(2) - if require_protocol and not proto: - return url # not protocol, no linkify - - if proto and proto not in permitted_protocols: - return url # bad protocol, no linkify - - href = m.group(1) - if not proto: - href = "http://" + href # no proto specified, use http - - if callable(extra_params): - params = " " + extra_params(href).strip() - else: - params = extra_params - - # clip long urls. max_len is just an approximation - max_len = 30 - if shorten and len(url) > max_len: - before_clip = url - if proto: - proto_len = len(proto) + 1 + len(m.group(3) or "") # +1 for : - else: - proto_len = 0 - - parts = url[proto_len:].split("/") - if len(parts) > 1: - # Grab the whole host part plus the first bit of the path - # The path is usually not that interesting once shortened - # (no more slug, etc), so it really just provides a little - # extra indication of shortening. - url = url[:proto_len] + parts[0] + "/" + \ - parts[1][:8].split('?')[0].split('.')[0] - - if len(url) > max_len * 1.5: # still too long - url = url[:max_len] - - if url != before_clip: - amp = url.rfind('&') - # avoid splitting html char entities - if amp > max_len - 5: - url = url[:amp] - url += "..." - - if len(url) >= len(before_clip): - url = before_clip - else: - # full url is visible on mouse-over (for those who don't - # have a status bar, such as Safari by default) - params += ' title="%s"' % href - - return u'<a href="%s"%s>%s</a>' % (href, params, url) - - # First HTML-escape so that our strings are all safe. - # The regex is modified to avoid character entites other than & so - # that we won't pick up ", etc. - text = _unicode(xhtml_escape(text)) - return _URL_RE.sub(make_link, text) - - -def _convert_entity(m): - if m.group(1) == "#": - try: - if m.group(2)[:1].lower() == 'x': - return unichr(int(m.group(2)[1:], 16)) - else: - return unichr(int(m.group(2))) - except ValueError: - return "&#%s;" % m.group(2) - try: - return _HTML_UNICODE_MAP[m.group(2)] - except KeyError: - return "&%s;" % m.group(2) - - -def _build_unicode_map(): - unicode_map = {} - for name, value in htmlentitydefs.name2codepoint.items(): - unicode_map[name] = unichr(value) - return unicode_map - - -_HTML_UNICODE_MAP = _build_unicode_map() diff --git a/lib/tornado/gen.py b/lib/tornado/gen.py deleted file mode 100755 index 3556374d..00000000 --- a/lib/tornado/gen.py +++ /dev/null @@ -1,1367 +0,0 @@ -"""``tornado.gen`` implements generator-based coroutines. - -.. note:: - - The "decorator and generator" approach in this module is a - precursor to native coroutines (using ``async def`` and ``await``) - which were introduced in Python 3.5. Applications that do not - require compatibility with older versions of Python should use - native coroutines instead. Some parts of this module are still - useful with native coroutines, notably `multi`, `sleep`, - `WaitIterator`, and `with_timeout`. Some of these functions have - counterparts in the `asyncio` module which may be used as well, - although the two may not necessarily be 100% compatible. - -Coroutines provide an easier way to work in an asynchronous -environment than chaining callbacks. Code using coroutines is -technically asynchronous, but it is written as a single generator -instead of a collection of separate functions. - -For example, the following callback-based asynchronous handler: - -.. testcode:: - - class AsyncHandler(RequestHandler): - @asynchronous - def get(self): - http_client = AsyncHTTPClient() - http_client.fetch("http://example.com", - callback=self.on_fetch) - - def on_fetch(self, response): - do_something_with_response(response) - self.render("template.html") - -.. testoutput:: - :hide: - -could be written with ``gen`` as: - -.. testcode:: - - class GenAsyncHandler(RequestHandler): - @gen.coroutine - def get(self): - http_client = AsyncHTTPClient() - response = yield http_client.fetch("http://example.com") - do_something_with_response(response) - self.render("template.html") - -.. testoutput:: - :hide: - -Most asynchronous functions in Tornado return a `.Future`; -yielding this object returns its ``Future.result``. - -You can also yield a list or dict of ``Futures``, which will be -started at the same time and run in parallel; a list or dict of results will -be returned when they are all finished: - -.. testcode:: - - @gen.coroutine - def get(self): - http_client = AsyncHTTPClient() - response1, response2 = yield [http_client.fetch(url1), - http_client.fetch(url2)] - response_dict = yield dict(response3=http_client.fetch(url3), - response4=http_client.fetch(url4)) - response3 = response_dict['response3'] - response4 = response_dict['response4'] - -.. testoutput:: - :hide: - -If the `~functools.singledispatch` library is available (standard in -Python 3.4, available via the `singledispatch -<https://pypi.python.org/pypi/singledispatch>`_ package on older -versions), additional types of objects may be yielded. Tornado includes -support for ``asyncio.Future`` and Twisted's ``Deferred`` class when -``tornado.platform.asyncio`` and ``tornado.platform.twisted`` are imported. -See the `convert_yielded` function to extend this mechanism. - -.. versionchanged:: 3.2 - Dict support added. - -.. versionchanged:: 4.1 - Support added for yielding ``asyncio`` Futures and Twisted Deferreds - via ``singledispatch``. - -""" -from __future__ import absolute_import, division, print_function - -import collections -import functools -import itertools -import os -import sys -import types -import warnings - -from tornado.concurrent import (Future, is_future, chain_future, future_set_exc_info, - future_add_done_callback, future_set_result_unless_cancelled) -from tornado.ioloop import IOLoop -from tornado.log import app_log -from tornado import stack_context -from tornado.util import PY3, raise_exc_info, TimeoutError - -try: - try: - # py34+ - from functools import singledispatch # type: ignore - except ImportError: - from singledispatch import singledispatch # backport -except ImportError: - # In most cases, singledispatch is required (to avoid - # difficult-to-diagnose problems in which the functionality - # available differs depending on which invisble packages are - # installed). However, in Google App Engine third-party - # dependencies are more trouble so we allow this module to be - # imported without it. - if 'APPENGINE_RUNTIME' not in os.environ: - raise - singledispatch = None - -try: - try: - # py35+ - from collections.abc import Generator as GeneratorType # type: ignore - except ImportError: - from backports_abc import Generator as GeneratorType # type: ignore - - try: - # py35+ - from inspect import isawaitable # type: ignore - except ImportError: - from backports_abc import isawaitable -except ImportError: - if 'APPENGINE_RUNTIME' not in os.environ: - raise - from types import GeneratorType - - def isawaitable(x): # type: ignore - return False - -if PY3: - import builtins -else: - import __builtin__ as builtins - - -class KeyReuseError(Exception): - pass - - -class UnknownKeyError(Exception): - pass - - -class LeakedCallbackError(Exception): - pass - - -class BadYieldError(Exception): - pass - - -class ReturnValueIgnoredError(Exception): - pass - - -def _value_from_stopiteration(e): - try: - # StopIteration has a value attribute beginning in py33. - # So does our Return class. - return e.value - except AttributeError: - pass - try: - # Cython backports coroutine functionality by putting the value in - # e.args[0]. - return e.args[0] - except (AttributeError, IndexError): - return None - - -def _create_future(): - future = Future() - # Fixup asyncio debug info by removing extraneous stack entries - source_traceback = getattr(future, "_source_traceback", ()) - while source_traceback: - # Each traceback entry is equivalent to a - # (filename, self.lineno, self.name, self.line) tuple - filename = source_traceback[-1][0] - if filename == __file__: - del source_traceback[-1] - else: - break - return future - - -def engine(func): - """Callback-oriented decorator for asynchronous generators. - - This is an older interface; for new code that does not need to be - compatible with versions of Tornado older than 3.0 the - `coroutine` decorator is recommended instead. - - This decorator is similar to `coroutine`, except it does not - return a `.Future` and the ``callback`` argument is not treated - specially. - - In most cases, functions decorated with `engine` should take - a ``callback`` argument and invoke it with their result when - they are finished. One notable exception is the - `~tornado.web.RequestHandler` :ref:`HTTP verb methods <verbs>`, - which use ``self.finish()`` in place of a callback argument. - - .. deprecated:: 5.1 - - This decorator will be removed in 6.0. Use `coroutine` or - ``async def`` instead. - """ - warnings.warn("gen.engine is deprecated, use gen.coroutine or async def instead", - DeprecationWarning) - func = _make_coroutine_wrapper(func, replace_callback=False) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - future = func(*args, **kwargs) - - def final_callback(future): - if future.result() is not None: - raise ReturnValueIgnoredError( - "@gen.engine functions cannot return values: %r" % - (future.result(),)) - # The engine interface doesn't give us any way to return - # errors but to raise them into the stack context. - # Save the stack context here to use when the Future has resolved. - future_add_done_callback(future, stack_context.wrap(final_callback)) - return wrapper - - -def coroutine(func): - """Decorator for asynchronous generators. - - Any generator that yields objects from this module must be wrapped - in either this decorator or `engine`. - - Coroutines may "return" by raising the special exception - `Return(value) <Return>`. In Python 3.3+, it is also possible for - the function to simply use the ``return value`` statement (prior to - Python 3.3 generators were not allowed to also return values). - In all versions of Python a coroutine that simply wishes to exit - early may use the ``return`` statement without a value. - - Functions with this decorator return a `.Future`. Additionally, - they may be called with a ``callback`` keyword argument, which - will be invoked with the future's result when it resolves. If the - coroutine fails, the callback will not be run and an exception - will be raised into the surrounding `.StackContext`. The - ``callback`` argument is not visible inside the decorated - function; it is handled by the decorator itself. - - .. warning:: - - When exceptions occur inside a coroutine, the exception - information will be stored in the `.Future` object. You must - examine the result of the `.Future` object, or the exception - may go unnoticed by your code. This means yielding the function - if called from another coroutine, using something like - `.IOLoop.run_sync` for top-level calls, or passing the `.Future` - to `.IOLoop.add_future`. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - return _make_coroutine_wrapper(func, replace_callback=True) - - -def _make_coroutine_wrapper(func, replace_callback): - """The inner workings of ``@gen.coroutine`` and ``@gen.engine``. - - The two decorators differ in their treatment of the ``callback`` - argument, so we cannot simply implement ``@engine`` in terms of - ``@coroutine``. - """ - # On Python 3.5, set the coroutine flag on our generator, to allow it - # to be used with 'await'. - wrapped = func - if hasattr(types, 'coroutine'): - func = types.coroutine(func) - - @functools.wraps(wrapped) - def wrapper(*args, **kwargs): - future = _create_future() - - if replace_callback and 'callback' in kwargs: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning, stacklevel=2) - callback = kwargs.pop('callback') - IOLoop.current().add_future( - future, lambda future: callback(future.result())) - - try: - result = func(*args, **kwargs) - except (Return, StopIteration) as e: - result = _value_from_stopiteration(e) - except Exception: - future_set_exc_info(future, sys.exc_info()) - try: - return future - finally: - # Avoid circular references - future = None - else: - if isinstance(result, GeneratorType): - # Inline the first iteration of Runner.run. This lets us - # avoid the cost of creating a Runner when the coroutine - # never actually yields, which in turn allows us to - # use "optional" coroutines in critical path code without - # performance penalty for the synchronous case. - try: - orig_stack_contexts = stack_context._state.contexts - yielded = next(result) - if stack_context._state.contexts is not orig_stack_contexts: - yielded = _create_future() - yielded.set_exception( - stack_context.StackContextInconsistentError( - 'stack_context inconsistency (probably caused ' - 'by yield within a "with StackContext" block)')) - except (StopIteration, Return) as e: - future_set_result_unless_cancelled(future, _value_from_stopiteration(e)) - except Exception: - future_set_exc_info(future, sys.exc_info()) - else: - # Provide strong references to Runner objects as long - # as their result future objects also have strong - # references (typically from the parent coroutine's - # Runner). This keeps the coroutine's Runner alive. - # We do this by exploiting the public API - # add_done_callback() instead of putting a private - # attribute on the Future. - # (Github issues #1769, #2229). - runner = Runner(result, future, yielded) - future.add_done_callback(lambda _: runner) - yielded = None - try: - return future - finally: - # Subtle memory optimization: if next() raised an exception, - # the future's exc_info contains a traceback which - # includes this stack frame. This creates a cycle, - # which will be collected at the next full GC but has - # been shown to greatly increase memory usage of - # benchmarks (relative to the refcount-based scheme - # used in the absence of cycles). We can avoid the - # cycle by clearing the local variable after we return it. - future = None - future_set_result_unless_cancelled(future, result) - return future - - wrapper.__wrapped__ = wrapped - wrapper.__tornado_coroutine__ = True - return wrapper - - -def is_coroutine_function(func): - """Return whether *func* is a coroutine function, i.e. a function - wrapped with `~.gen.coroutine`. - - .. versionadded:: 4.5 - """ - return getattr(func, '__tornado_coroutine__', False) - - -class Return(Exception): - """Special exception to return a value from a `coroutine`. - - If this exception is raised, its value argument is used as the - result of the coroutine:: - - @gen.coroutine - def fetch_json(url): - response = yield AsyncHTTPClient().fetch(url) - raise gen.Return(json_decode(response.body)) - - In Python 3.3, this exception is no longer necessary: the ``return`` - statement can be used directly to return a value (previously - ``yield`` and ``return`` with a value could not be combined in the - same function). - - By analogy with the return statement, the value argument is optional, - but it is never necessary to ``raise gen.Return()``. The ``return`` - statement can be used with no arguments instead. - """ - def __init__(self, value=None): - super(Return, self).__init__() - self.value = value - # Cython recognizes subclasses of StopIteration with a .args tuple. - self.args = (value,) - - -class WaitIterator(object): - """Provides an iterator to yield the results of futures as they finish. - - Yielding a set of futures like this: - - ``results = yield [future1, future2]`` - - pauses the coroutine until both ``future1`` and ``future2`` - return, and then restarts the coroutine with the results of both - futures. If either future is an exception, the expression will - raise that exception and all the results will be lost. - - If you need to get the result of each future as soon as possible, - or if you need the result of some futures even if others produce - errors, you can use ``WaitIterator``:: - - wait_iterator = gen.WaitIterator(future1, future2) - while not wait_iterator.done(): - try: - result = yield wait_iterator.next() - except Exception as e: - print("Error {} from {}".format(e, wait_iterator.current_future)) - else: - print("Result {} received from {} at {}".format( - result, wait_iterator.current_future, - wait_iterator.current_index)) - - Because results are returned as soon as they are available the - output from the iterator *will not be in the same order as the - input arguments*. If you need to know which future produced the - current result, you can use the attributes - ``WaitIterator.current_future``, or ``WaitIterator.current_index`` - to get the index of the future from the input list. (if keyword - arguments were used in the construction of the `WaitIterator`, - ``current_index`` will use the corresponding keyword). - - On Python 3.5, `WaitIterator` implements the async iterator - protocol, so it can be used with the ``async for`` statement (note - that in this version the entire iteration is aborted if any value - raises an exception, while the previous example can continue past - individual errors):: - - async for result in gen.WaitIterator(future1, future2): - print("Result {} received from {} at {}".format( - result, wait_iterator.current_future, - wait_iterator.current_index)) - - .. versionadded:: 4.1 - - .. versionchanged:: 4.3 - Added ``async for`` support in Python 3.5. - - """ - def __init__(self, *args, **kwargs): - if args and kwargs: - raise ValueError( - "You must provide args or kwargs, not both") - - if kwargs: - self._unfinished = dict((f, k) for (k, f) in kwargs.items()) - futures = list(kwargs.values()) - else: - self._unfinished = dict((f, i) for (i, f) in enumerate(args)) - futures = args - - self._finished = collections.deque() - self.current_index = self.current_future = None - self._running_future = None - - for future in futures: - future_add_done_callback(future, self._done_callback) - - def done(self): - """Returns True if this iterator has no more results.""" - if self._finished or self._unfinished: - return False - # Clear the 'current' values when iteration is done. - self.current_index = self.current_future = None - return True - - def next(self): - """Returns a `.Future` that will yield the next available result. - - Note that this `.Future` will not be the same object as any of - the inputs. - """ - self._running_future = Future() - - if self._finished: - self._return_result(self._finished.popleft()) - - return self._running_future - - def _done_callback(self, done): - if self._running_future and not self._running_future.done(): - self._return_result(done) - else: - self._finished.append(done) - - def _return_result(self, done): - """Called set the returned future's state that of the future - we yielded, and set the current future for the iterator. - """ - chain_future(done, self._running_future) - - self.current_future = done - self.current_index = self._unfinished.pop(done) - - def __aiter__(self): - return self - - def __anext__(self): - if self.done(): - # Lookup by name to silence pyflakes on older versions. - raise getattr(builtins, 'StopAsyncIteration')() - return self.next() - - -class YieldPoint(object): - """Base class for objects that may be yielded from the generator. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. This class and all its subclasses - will be removed in 6.0 - """ - def __init__(self): - warnings.warn("YieldPoint is deprecated, use Futures instead", - DeprecationWarning) - - def start(self, runner): - """Called by the runner after the generator has yielded. - - No other methods will be called on this object before ``start``. - """ - raise NotImplementedError() - - def is_ready(self): - """Called by the runner to determine whether to resume the generator. - - Returns a boolean; may be called more than once. - """ - raise NotImplementedError() - - def get_result(self): - """Returns the value to use as the result of the yield expression. - - This method will only be called once, and only after `is_ready` - has returned true. - """ - raise NotImplementedError() - - -class Callback(YieldPoint): - """Returns a callable object that will allow a matching `Wait` to proceed. - - The key may be any value suitable for use as a dictionary key, and is - used to match ``Callbacks`` to their corresponding ``Waits``. The key - must be unique among outstanding callbacks within a single run of the - generator function, but may be reused across different runs of the same - function (so constants generally work fine). - - The callback may be called with zero or one arguments; if an argument - is given it will be returned by `Wait`. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. This class will be removed in 6.0. - """ - def __init__(self, key): - warnings.warn("gen.Callback is deprecated, use Futures instead", - DeprecationWarning) - self.key = key - - def start(self, runner): - self.runner = runner - runner.register_callback(self.key) - - def is_ready(self): - return True - - def get_result(self): - return self.runner.result_callback(self.key) - - -class Wait(YieldPoint): - """Returns the argument passed to the result of a previous `Callback`. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. This class will be removed in 6.0. - """ - def __init__(self, key): - warnings.warn("gen.Wait is deprecated, use Futures instead", - DeprecationWarning) - self.key = key - - def start(self, runner): - self.runner = runner - - def is_ready(self): - return self.runner.is_ready(self.key) - - def get_result(self): - return self.runner.pop_result(self.key) - - -class WaitAll(YieldPoint): - """Returns the results of multiple previous `Callbacks <Callback>`. - - The argument is a sequence of `Callback` keys, and the result is - a list of results in the same order. - - `WaitAll` is equivalent to yielding a list of `Wait` objects. - - .. deprecated:: 4.0 - Use `Futures <.Future>` instead. This class will be removed in 6.0. - """ - def __init__(self, keys): - warnings.warn("gen.WaitAll is deprecated, use gen.multi instead", - DeprecationWarning) - self.keys = keys - - def start(self, runner): - self.runner = runner - - def is_ready(self): - return all(self.runner.is_ready(key) for key in self.keys) - - def get_result(self): - return [self.runner.pop_result(key) for key in self.keys] - - -def Task(func, *args, **kwargs): - """Adapts a callback-based asynchronous function for use in coroutines. - - Takes a function (and optional additional arguments) and runs it with - those arguments plus a ``callback`` keyword argument. The argument passed - to the callback is returned as the result of the yield expression. - - .. versionchanged:: 4.0 - ``gen.Task`` is now a function that returns a `.Future`, instead of - a subclass of `YieldPoint`. It still behaves the same way when - yielded. - - .. deprecated:: 5.1 - This function is deprecated and will be removed in 6.0. - """ - warnings.warn("gen.Task is deprecated, use Futures instead", - DeprecationWarning) - future = _create_future() - - def handle_exception(typ, value, tb): - if future.done(): - return False - future_set_exc_info(future, (typ, value, tb)) - return True - - def set_result(result): - if future.done(): - return - future_set_result_unless_cancelled(future, result) - with stack_context.ExceptionStackContext(handle_exception): - func(*args, callback=_argument_adapter(set_result), **kwargs) - return future - - -class YieldFuture(YieldPoint): - def __init__(self, future): - """Adapts a `.Future` to the `YieldPoint` interface. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - .. deprecated:: 5.1 - This class will be removed in 6.0. - """ - warnings.warn("YieldFuture is deprecated, use Futures instead", - DeprecationWarning) - self.future = future - self.io_loop = IOLoop.current() - - def start(self, runner): - if not self.future.done(): - self.runner = runner - self.key = object() - runner.register_callback(self.key) - self.io_loop.add_future(self.future, runner.result_callback(self.key)) - else: - self.runner = None - self.result_fn = self.future.result - - def is_ready(self): - if self.runner is not None: - return self.runner.is_ready(self.key) - else: - return True - - def get_result(self): - if self.runner is not None: - return self.runner.pop_result(self.key).result() - else: - return self.result_fn() - - -def _contains_yieldpoint(children): - """Returns True if ``children`` contains any YieldPoints. - - ``children`` may be a dict or a list, as used by `MultiYieldPoint` - and `multi_future`. - """ - if isinstance(children, dict): - return any(isinstance(i, YieldPoint) for i in children.values()) - if isinstance(children, list): - return any(isinstance(i, YieldPoint) for i in children) - return False - - -def multi(children, quiet_exceptions=()): - """Runs multiple asynchronous operations in parallel. - - ``children`` may either be a list or a dict whose values are - yieldable objects. ``multi()`` returns a new yieldable - object that resolves to a parallel structure containing their - results. If ``children`` is a list, the result is a list of - results in the same order; if it is a dict, the result is a dict - with the same keys. - - That is, ``results = yield multi(list_of_futures)`` is equivalent - to:: - - results = [] - for future in list_of_futures: - results.append(yield future) - - If any children raise exceptions, ``multi()`` will raise the first - one. All others will be logged, unless they are of types - contained in the ``quiet_exceptions`` argument. - - If any of the inputs are `YieldPoints <YieldPoint>`, the returned - yieldable object is a `YieldPoint`. Otherwise, returns a `.Future`. - This means that the result of `multi` can be used in a native - coroutine if and only if all of its children can be. - - In a ``yield``-based coroutine, it is not normally necessary to - call this function directly, since the coroutine runner will - do it automatically when a list or dict is yielded. However, - it is necessary in ``await``-based coroutines, or to pass - the ``quiet_exceptions`` argument. - - This function is available under the names ``multi()`` and ``Multi()`` - for historical reasons. - - Cancelling a `.Future` returned by ``multi()`` does not cancel its - children. `asyncio.gather` is similar to ``multi()``, but it does - cancel its children. - - .. versionchanged:: 4.2 - If multiple yieldables fail, any exceptions after the first - (which is raised) will be logged. Added the ``quiet_exceptions`` - argument to suppress this logging for selected exception types. - - .. versionchanged:: 4.3 - Replaced the class ``Multi`` and the function ``multi_future`` - with a unified function ``multi``. Added support for yieldables - other than `YieldPoint` and `.Future`. - - """ - if _contains_yieldpoint(children): - return MultiYieldPoint(children, quiet_exceptions=quiet_exceptions) - else: - return multi_future(children, quiet_exceptions=quiet_exceptions) - - -Multi = multi - - -class MultiYieldPoint(YieldPoint): - """Runs multiple asynchronous operations in parallel. - - This class is similar to `multi`, but it always creates a stack - context even when no children require it. It is not compatible with - native coroutines. - - .. versionchanged:: 4.2 - If multiple ``YieldPoints`` fail, any exceptions after the first - (which is raised) will be logged. Added the ``quiet_exceptions`` - argument to suppress this logging for selected exception types. - - .. versionchanged:: 4.3 - Renamed from ``Multi`` to ``MultiYieldPoint``. The name ``Multi`` - remains as an alias for the equivalent `multi` function. - - .. deprecated:: 4.3 - Use `multi` instead. This class will be removed in 6.0. - """ - def __init__(self, children, quiet_exceptions=()): - warnings.warn("MultiYieldPoint is deprecated, use Futures instead", - DeprecationWarning) - self.keys = None - if isinstance(children, dict): - self.keys = list(children.keys()) - children = children.values() - self.children = [] - for i in children: - if not isinstance(i, YieldPoint): - i = convert_yielded(i) - if is_future(i): - i = YieldFuture(i) - self.children.append(i) - assert all(isinstance(i, YieldPoint) for i in self.children) - self.unfinished_children = set(self.children) - self.quiet_exceptions = quiet_exceptions - - def start(self, runner): - for i in self.children: - i.start(runner) - - def is_ready(self): - finished = list(itertools.takewhile( - lambda i: i.is_ready(), self.unfinished_children)) - self.unfinished_children.difference_update(finished) - return not self.unfinished_children - - def get_result(self): - result_list = [] - exc_info = None - for f in self.children: - try: - result_list.append(f.get_result()) - except Exception as e: - if exc_info is None: - exc_info = sys.exc_info() - else: - if not isinstance(e, self.quiet_exceptions): - app_log.error("Multiple exceptions in yield list", - exc_info=True) - if exc_info is not None: - raise_exc_info(exc_info) - if self.keys is not None: - return dict(zip(self.keys, result_list)) - else: - return list(result_list) - - -def multi_future(children, quiet_exceptions=()): - """Wait for multiple asynchronous futures in parallel. - - This function is similar to `multi`, but does not support - `YieldPoints <YieldPoint>`. - - .. versionadded:: 4.0 - - .. versionchanged:: 4.2 - If multiple ``Futures`` fail, any exceptions after the first (which is - raised) will be logged. Added the ``quiet_exceptions`` - argument to suppress this logging for selected exception types. - - .. deprecated:: 4.3 - Use `multi` instead. - """ - if isinstance(children, dict): - keys = list(children.keys()) - children = children.values() - else: - keys = None - children = list(map(convert_yielded, children)) - assert all(is_future(i) or isinstance(i, _NullFuture) for i in children) - unfinished_children = set(children) - - future = _create_future() - if not children: - future_set_result_unless_cancelled(future, - {} if keys is not None else []) - - def callback(f): - unfinished_children.remove(f) - if not unfinished_children: - result_list = [] - for f in children: - try: - result_list.append(f.result()) - except Exception as e: - if future.done(): - if not isinstance(e, quiet_exceptions): - app_log.error("Multiple exceptions in yield list", - exc_info=True) - else: - future_set_exc_info(future, sys.exc_info()) - if not future.done(): - if keys is not None: - future_set_result_unless_cancelled(future, - dict(zip(keys, result_list))) - else: - future_set_result_unless_cancelled(future, result_list) - - listening = set() - for f in children: - if f not in listening: - listening.add(f) - future_add_done_callback(f, callback) - return future - - -def maybe_future(x): - """Converts ``x`` into a `.Future`. - - If ``x`` is already a `.Future`, it is simply returned; otherwise - it is wrapped in a new `.Future`. This is suitable for use as - ``result = yield gen.maybe_future(f())`` when you don't know whether - ``f()`` returns a `.Future` or not. - - .. deprecated:: 4.3 - This function only handles ``Futures``, not other yieldable objects. - Instead of `maybe_future`, check for the non-future result types - you expect (often just ``None``), and ``yield`` anything unknown. - """ - if is_future(x): - return x - else: - fut = _create_future() - fut.set_result(x) - return fut - - -def with_timeout(timeout, future, quiet_exceptions=()): - """Wraps a `.Future` (or other yieldable object) in a timeout. - - Raises `tornado.util.TimeoutError` if the input future does not - complete before ``timeout``, which may be specified in any form - allowed by `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or - an absolute time relative to `.IOLoop.time`) - - If the wrapped `.Future` fails after it has timed out, the exception - will be logged unless it is of a type contained in ``quiet_exceptions`` - (which may be an exception type or a sequence of types). - - Does not support `YieldPoint` subclasses. - - The wrapped `.Future` is not canceled when the timeout expires, - permitting it to be reused. `asyncio.wait_for` is similar to this - function but it does cancel the wrapped `.Future` on timeout. - - .. versionadded:: 4.0 - - .. versionchanged:: 4.1 - Added the ``quiet_exceptions`` argument and the logging of unhandled - exceptions. - - .. versionchanged:: 4.4 - Added support for yieldable objects other than `.Future`. - - """ - # TODO: allow YieldPoints in addition to other yieldables? - # Tricky to do with stack_context semantics. - # - # It's tempting to optimize this by cancelling the input future on timeout - # instead of creating a new one, but A) we can't know if we are the only - # one waiting on the input future, so cancelling it might disrupt other - # callers and B) concurrent futures can only be cancelled while they are - # in the queue, so cancellation cannot reliably bound our waiting time. - future = convert_yielded(future) - result = _create_future() - chain_future(future, result) - io_loop = IOLoop.current() - - def error_callback(future): - try: - future.result() - except Exception as e: - if not isinstance(e, quiet_exceptions): - app_log.error("Exception in Future %r after timeout", - future, exc_info=True) - - def timeout_callback(): - if not result.done(): - result.set_exception(TimeoutError("Timeout")) - # In case the wrapped future goes on to fail, log it. - future_add_done_callback(future, error_callback) - timeout_handle = io_loop.add_timeout( - timeout, timeout_callback) - if isinstance(future, Future): - # We know this future will resolve on the IOLoop, so we don't - # need the extra thread-safety of IOLoop.add_future (and we also - # don't care about StackContext here. - future_add_done_callback( - future, lambda future: io_loop.remove_timeout(timeout_handle)) - else: - # concurrent.futures.Futures may resolve on any thread, so we - # need to route them back to the IOLoop. - io_loop.add_future( - future, lambda future: io_loop.remove_timeout(timeout_handle)) - return result - - -def sleep(duration): - """Return a `.Future` that resolves after the given number of seconds. - - When used with ``yield`` in a coroutine, this is a non-blocking - analogue to `time.sleep` (which should not be used in coroutines - because it is blocking):: - - yield gen.sleep(0.5) - - Note that calling this function on its own does nothing; you must - wait on the `.Future` it returns (usually by yielding it). - - .. versionadded:: 4.1 - """ - f = _create_future() - IOLoop.current().call_later(duration, - lambda: future_set_result_unless_cancelled(f, None)) - return f - - -class _NullFuture(object): - """_NullFuture resembles a Future that finished with a result of None. - - It's not actually a `Future` to avoid depending on a particular event loop. - Handled as a special case in the coroutine runner. - """ - def result(self): - return None - - def done(self): - return True - - -# _null_future is used as a dummy value in the coroutine runner. It differs -# from moment in that moment always adds a delay of one IOLoop iteration -# while _null_future is processed as soon as possible. -_null_future = _NullFuture() - -moment = _NullFuture() -moment.__doc__ = \ - """A special object which may be yielded to allow the IOLoop to run for -one iteration. - -This is not needed in normal use but it can be helpful in long-running -coroutines that are likely to yield Futures that are ready instantly. - -Usage: ``yield gen.moment`` - -.. versionadded:: 4.0 - -.. deprecated:: 4.5 - ``yield None`` (or ``yield`` with no argument) is now equivalent to - ``yield gen.moment``. -""" - - -class Runner(object): - """Internal implementation of `tornado.gen.engine`. - - Maintains information about pending callbacks and their results. - - The results of the generator are stored in ``result_future`` (a - `.Future`) - """ - def __init__(self, gen, result_future, first_yielded): - self.gen = gen - self.result_future = result_future - self.future = _null_future - self.yield_point = None - self.pending_callbacks = None - self.results = None - self.running = False - self.finished = False - self.had_exception = False - self.io_loop = IOLoop.current() - # For efficiency, we do not create a stack context until we - # reach a YieldPoint (stack contexts are required for the historical - # semantics of YieldPoints, but not for Futures). When we have - # done so, this field will be set and must be called at the end - # of the coroutine. - self.stack_context_deactivate = None - if self.handle_yield(first_yielded): - gen = result_future = first_yielded = None - self.run() - - def register_callback(self, key): - """Adds ``key`` to the list of callbacks.""" - if self.pending_callbacks is None: - # Lazily initialize the old-style YieldPoint data structures. - self.pending_callbacks = set() - self.results = {} - if key in self.pending_callbacks: - raise KeyReuseError("key %r is already pending" % (key,)) - self.pending_callbacks.add(key) - - def is_ready(self, key): - """Returns true if a result is available for ``key``.""" - if self.pending_callbacks is None or key not in self.pending_callbacks: - raise UnknownKeyError("key %r is not pending" % (key,)) - return key in self.results - - def set_result(self, key, result): - """Sets the result for ``key`` and attempts to resume the generator.""" - self.results[key] = result - if self.yield_point is not None and self.yield_point.is_ready(): - try: - future_set_result_unless_cancelled(self.future, - self.yield_point.get_result()) - except: - future_set_exc_info(self.future, sys.exc_info()) - self.yield_point = None - self.run() - - def pop_result(self, key): - """Returns the result for ``key`` and unregisters it.""" - self.pending_callbacks.remove(key) - return self.results.pop(key) - - def run(self): - """Starts or resumes the generator, running until it reaches a - yield point that is not ready. - """ - if self.running or self.finished: - return - try: - self.running = True - while True: - future = self.future - if not future.done(): - return - self.future = None - try: - orig_stack_contexts = stack_context._state.contexts - exc_info = None - - try: - value = future.result() - except Exception: - self.had_exception = True - exc_info = sys.exc_info() - future = None - - if exc_info is not None: - try: - yielded = self.gen.throw(*exc_info) - finally: - # Break up a reference to itself - # for faster GC on CPython. - exc_info = None - else: - yielded = self.gen.send(value) - - if stack_context._state.contexts is not orig_stack_contexts: - self.gen.throw( - stack_context.StackContextInconsistentError( - 'stack_context inconsistency (probably caused ' - 'by yield within a "with StackContext" block)')) - except (StopIteration, Return) as e: - self.finished = True - self.future = _null_future - if self.pending_callbacks and not self.had_exception: - # If we ran cleanly without waiting on all callbacks - # raise an error (really more of a warning). If we - # had an exception then some callbacks may have been - # orphaned, so skip the check in that case. - raise LeakedCallbackError( - "finished without waiting for callbacks %r" % - self.pending_callbacks) - future_set_result_unless_cancelled(self.result_future, - _value_from_stopiteration(e)) - self.result_future = None - self._deactivate_stack_context() - return - except Exception: - self.finished = True - self.future = _null_future - future_set_exc_info(self.result_future, sys.exc_info()) - self.result_future = None - self._deactivate_stack_context() - return - if not self.handle_yield(yielded): - return - yielded = None - finally: - self.running = False - - def handle_yield(self, yielded): - # Lists containing YieldPoints require stack contexts; - # other lists are handled in convert_yielded. - if _contains_yieldpoint(yielded): - yielded = multi(yielded) - - if isinstance(yielded, YieldPoint): - # YieldPoints are too closely coupled to the Runner to go - # through the generic convert_yielded mechanism. - self.future = Future() - - def start_yield_point(): - try: - yielded.start(self) - if yielded.is_ready(): - future_set_result_unless_cancelled(self.future, yielded.get_result()) - else: - self.yield_point = yielded - except Exception: - self.future = Future() - future_set_exc_info(self.future, sys.exc_info()) - - if self.stack_context_deactivate is None: - # Start a stack context if this is the first - # YieldPoint we've seen. - with stack_context.ExceptionStackContext( - self.handle_exception) as deactivate: - self.stack_context_deactivate = deactivate - - def cb(): - start_yield_point() - self.run() - self.io_loop.add_callback(cb) - return False - else: - start_yield_point() - else: - try: - self.future = convert_yielded(yielded) - except BadYieldError: - self.future = Future() - future_set_exc_info(self.future, sys.exc_info()) - - if self.future is moment: - self.io_loop.add_callback(self.run) - return False - elif not self.future.done(): - def inner(f): - # Break a reference cycle to speed GC. - f = None # noqa - self.run() - self.io_loop.add_future( - self.future, inner) - return False - return True - - def result_callback(self, key): - return stack_context.wrap(_argument_adapter( - functools.partial(self.set_result, key))) - - def handle_exception(self, typ, value, tb): - if not self.running and not self.finished: - self.future = Future() - future_set_exc_info(self.future, (typ, value, tb)) - self.run() - return True - else: - return False - - def _deactivate_stack_context(self): - if self.stack_context_deactivate is not None: - self.stack_context_deactivate() - self.stack_context_deactivate = None - - -Arguments = collections.namedtuple('Arguments', ['args', 'kwargs']) - - -def _argument_adapter(callback): - """Returns a function that when invoked runs ``callback`` with one arg. - - If the function returned by this function is called with exactly - one argument, that argument is passed to ``callback``. Otherwise - the args tuple and kwargs dict are wrapped in an `Arguments` object. - """ - def wrapper(*args, **kwargs): - if kwargs or len(args) > 1: - callback(Arguments(args, kwargs)) - elif args: - callback(args[0]) - else: - callback(None) - return wrapper - - -# Convert Awaitables into Futures. -try: - import asyncio -except ImportError: - # Py2-compatible version for use with Cython. - # Copied from PEP 380. - @coroutine - def _wrap_awaitable(x): - if hasattr(x, '__await__'): - _i = x.__await__() - else: - _i = iter(x) - try: - _y = next(_i) - except StopIteration as _e: - _r = _value_from_stopiteration(_e) - else: - while 1: - try: - _s = yield _y - except GeneratorExit as _e: - try: - _m = _i.close - except AttributeError: - pass - else: - _m() - raise _e - except BaseException as _e: - _x = sys.exc_info() - try: - _m = _i.throw - except AttributeError: - raise _e - else: - try: - _y = _m(*_x) - except StopIteration as _e: - _r = _value_from_stopiteration(_e) - break - else: - try: - if _s is None: - _y = next(_i) - else: - _y = _i.send(_s) - except StopIteration as _e: - _r = _value_from_stopiteration(_e) - break - raise Return(_r) -else: - try: - _wrap_awaitable = asyncio.ensure_future - except AttributeError: - # asyncio.ensure_future was introduced in Python 3.4.4, but - # Debian jessie still ships with 3.4.2 so try the old name. - _wrap_awaitable = getattr(asyncio, 'async') - - -def convert_yielded(yielded): - """Convert a yielded object into a `.Future`. - - The default implementation accepts lists, dictionaries, and Futures. - - If the `~functools.singledispatch` library is available, this function - may be extended to support additional types. For example:: - - @convert_yielded.register(asyncio.Future) - def _(asyncio_future): - return tornado.platform.asyncio.to_tornado_future(asyncio_future) - - .. versionadded:: 4.1 - """ - # Lists and dicts containing YieldPoints were handled earlier. - if yielded is None or yielded is moment: - return moment - elif yielded is _null_future: - return _null_future - elif isinstance(yielded, (list, dict)): - return multi(yielded) - elif is_future(yielded): - return yielded - elif isawaitable(yielded): - return _wrap_awaitable(yielded) - else: - raise BadYieldError("yielded unknown object %r" % (yielded,)) - - -if singledispatch is not None: - convert_yielded = singledispatch(convert_yielded) diff --git a/lib/tornado/http1connection.py b/lib/tornado/http1connection.py deleted file mode 100755 index 6cc4071c..00000000 --- a/lib/tornado/http1connection.py +++ /dev/null @@ -1,751 +0,0 @@ -# -# Copyright 2014 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Client and server implementations of HTTP/1.x. - -.. versionadded:: 4.0 -""" - -from __future__ import absolute_import, division, print_function - -import re -import warnings - -from tornado.concurrent import (Future, future_add_done_callback, - future_set_result_unless_cancelled) -from tornado.escape import native_str, utf8 -from tornado import gen -from tornado import httputil -from tornado import iostream -from tornado.log import gen_log, app_log -from tornado import stack_context -from tornado.util import GzipDecompressor, PY3 - - -class _QuietException(Exception): - def __init__(self): - pass - - -class _ExceptionLoggingContext(object): - """Used with the ``with`` statement when calling delegate methods to - log any exceptions with the given logger. Any exceptions caught are - converted to _QuietException - """ - def __init__(self, logger): - self.logger = logger - - def __enter__(self): - pass - - def __exit__(self, typ, value, tb): - if value is not None: - self.logger.error("Uncaught exception", exc_info=(typ, value, tb)) - raise _QuietException - - -class HTTP1ConnectionParameters(object): - """Parameters for `.HTTP1Connection` and `.HTTP1ServerConnection`. - """ - def __init__(self, no_keep_alive=False, chunk_size=None, - max_header_size=None, header_timeout=None, max_body_size=None, - body_timeout=None, decompress=False): - """ - :arg bool no_keep_alive: If true, always close the connection after - one request. - :arg int chunk_size: how much data to read into memory at once - :arg int max_header_size: maximum amount of data for HTTP headers - :arg float header_timeout: how long to wait for all headers (seconds) - :arg int max_body_size: maximum amount of data for body - :arg float body_timeout: how long to wait while reading body (seconds) - :arg bool decompress: if true, decode incoming - ``Content-Encoding: gzip`` - """ - self.no_keep_alive = no_keep_alive - self.chunk_size = chunk_size or 65536 - self.max_header_size = max_header_size or 65536 - self.header_timeout = header_timeout - self.max_body_size = max_body_size - self.body_timeout = body_timeout - self.decompress = decompress - - -class HTTP1Connection(httputil.HTTPConnection): - """Implements the HTTP/1.x protocol. - - This class can be on its own for clients, or via `HTTP1ServerConnection` - for servers. - """ - def __init__(self, stream, is_client, params=None, context=None): - """ - :arg stream: an `.IOStream` - :arg bool is_client: client or server - :arg params: a `.HTTP1ConnectionParameters` instance or ``None`` - :arg context: an opaque application-defined object that can be accessed - as ``connection.context``. - """ - self.is_client = is_client - self.stream = stream - if params is None: - params = HTTP1ConnectionParameters() - self.params = params - self.context = context - self.no_keep_alive = params.no_keep_alive - # The body limits can be altered by the delegate, so save them - # here instead of just referencing self.params later. - self._max_body_size = (self.params.max_body_size or - self.stream.max_buffer_size) - self._body_timeout = self.params.body_timeout - # _write_finished is set to True when finish() has been called, - # i.e. there will be no more data sent. Data may still be in the - # stream's write buffer. - self._write_finished = False - # True when we have read the entire incoming body. - self._read_finished = False - # _finish_future resolves when all data has been written and flushed - # to the IOStream. - self._finish_future = Future() - # If true, the connection should be closed after this request - # (after the response has been written in the server side, - # and after it has been read in the client) - self._disconnect_on_finish = False - self._clear_callbacks() - # Save the start lines after we read or write them; they - # affect later processing (e.g. 304 responses and HEAD methods - # have content-length but no bodies) - self._request_start_line = None - self._response_start_line = None - self._request_headers = None - # True if we are writing output with chunked encoding. - self._chunking_output = None - # While reading a body with a content-length, this is the - # amount left to read. - self._expected_content_remaining = None - # A Future for our outgoing writes, returned by IOStream.write. - self._pending_write = None - - def read_response(self, delegate): - """Read a single HTTP response. - - Typical client-mode usage is to write a request using `write_headers`, - `write`, and `finish`, and then call ``read_response``. - - :arg delegate: a `.HTTPMessageDelegate` - - Returns a `.Future` that resolves to None after the full response has - been read. - """ - if self.params.decompress: - delegate = _GzipMessageDelegate(delegate, self.params.chunk_size) - return self._read_message(delegate) - - @gen.coroutine - def _read_message(self, delegate): - need_delegate_close = False - try: - header_future = self.stream.read_until_regex( - b"\r?\n\r?\n", - max_bytes=self.params.max_header_size) - if self.params.header_timeout is None: - header_data = yield header_future - else: - try: - header_data = yield gen.with_timeout( - self.stream.io_loop.time() + self.params.header_timeout, - header_future, - quiet_exceptions=iostream.StreamClosedError) - except gen.TimeoutError: - self.close() - raise gen.Return(False) - start_line, headers = self._parse_headers(header_data) - if self.is_client: - start_line = httputil.parse_response_start_line(start_line) - self._response_start_line = start_line - else: - start_line = httputil.parse_request_start_line(start_line) - self._request_start_line = start_line - self._request_headers = headers - - self._disconnect_on_finish = not self._can_keep_alive( - start_line, headers) - need_delegate_close = True - with _ExceptionLoggingContext(app_log): - header_future = delegate.headers_received(start_line, headers) - if header_future is not None: - yield header_future - if self.stream is None: - # We've been detached. - need_delegate_close = False - raise gen.Return(False) - skip_body = False - if self.is_client: - if (self._request_start_line is not None and - self._request_start_line.method == 'HEAD'): - skip_body = True - code = start_line.code - if code == 304: - # 304 responses may include the content-length header - # but do not actually have a body. - # http://tools.ietf.org/html/rfc7230#section-3.3 - skip_body = True - if code >= 100 and code < 200: - # 1xx responses should never indicate the presence of - # a body. - if ('Content-Length' in headers or - 'Transfer-Encoding' in headers): - raise httputil.HTTPInputError( - "Response code %d cannot have body" % code) - # TODO: client delegates will get headers_received twice - # in the case of a 100-continue. Document or change? - yield self._read_message(delegate) - else: - if (headers.get("Expect") == "100-continue" and - not self._write_finished): - self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") - if not skip_body: - body_future = self._read_body( - start_line.code if self.is_client else 0, headers, delegate) - if body_future is not None: - if self._body_timeout is None: - yield body_future - else: - try: - yield gen.with_timeout( - self.stream.io_loop.time() + self._body_timeout, - body_future, - quiet_exceptions=iostream.StreamClosedError) - except gen.TimeoutError: - gen_log.info("Timeout reading body from %s", - self.context) - self.stream.close() - raise gen.Return(False) - self._read_finished = True - if not self._write_finished or self.is_client: - need_delegate_close = False - with _ExceptionLoggingContext(app_log): - delegate.finish() - # If we're waiting for the application to produce an asynchronous - # response, and we're not detached, register a close callback - # on the stream (we didn't need one while we were reading) - if (not self._finish_future.done() and - self.stream is not None and - not self.stream.closed()): - self.stream.set_close_callback(self._on_connection_close) - yield self._finish_future - if self.is_client and self._disconnect_on_finish: - self.close() - if self.stream is None: - raise gen.Return(False) - except httputil.HTTPInputError as e: - gen_log.info("Malformed HTTP message from %s: %s", - self.context, e) - if not self.is_client: - yield self.stream.write(b'HTTP/1.1 400 Bad Request\r\n\r\n') - self.close() - raise gen.Return(False) - finally: - if need_delegate_close: - with _ExceptionLoggingContext(app_log): - delegate.on_connection_close() - header_future = None - self._clear_callbacks() - raise gen.Return(True) - - def _clear_callbacks(self): - """Clears the callback attributes. - - This allows the request handler to be garbage collected more - quickly in CPython by breaking up reference cycles. - """ - self._write_callback = None - self._write_future = None - self._close_callback = None - if self.stream is not None: - self.stream.set_close_callback(None) - - def set_close_callback(self, callback): - """Sets a callback that will be run when the connection is closed. - - Note that this callback is slightly different from - `.HTTPMessageDelegate.on_connection_close`: The - `.HTTPMessageDelegate` method is called when the connection is - closed while recieving a message. This callback is used when - there is not an active delegate (for example, on the server - side this callback is used if the client closes the connection - after sending its request but before receiving all the - response. - """ - self._close_callback = stack_context.wrap(callback) - - def _on_connection_close(self): - # Note that this callback is only registered on the IOStream - # when we have finished reading the request and are waiting for - # the application to produce its response. - if self._close_callback is not None: - callback = self._close_callback - self._close_callback = None - callback() - if not self._finish_future.done(): - future_set_result_unless_cancelled(self._finish_future, None) - self._clear_callbacks() - - def close(self): - if self.stream is not None: - self.stream.close() - self._clear_callbacks() - if not self._finish_future.done(): - future_set_result_unless_cancelled(self._finish_future, None) - - def detach(self): - """Take control of the underlying stream. - - Returns the underlying `.IOStream` object and stops all further - HTTP processing. May only be called during - `.HTTPMessageDelegate.headers_received`. Intended for implementing - protocols like websockets that tunnel over an HTTP handshake. - """ - self._clear_callbacks() - stream = self.stream - self.stream = None - if not self._finish_future.done(): - future_set_result_unless_cancelled(self._finish_future, None) - return stream - - def set_body_timeout(self, timeout): - """Sets the body timeout for a single request. - - Overrides the value from `.HTTP1ConnectionParameters`. - """ - self._body_timeout = timeout - - def set_max_body_size(self, max_body_size): - """Sets the body size limit for a single request. - - Overrides the value from `.HTTP1ConnectionParameters`. - """ - self._max_body_size = max_body_size - - def write_headers(self, start_line, headers, chunk=None, callback=None): - """Implements `.HTTPConnection.write_headers`.""" - lines = [] - if self.is_client: - self._request_start_line = start_line - lines.append(utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) - # Client requests with a non-empty body must have either a - # Content-Length or a Transfer-Encoding. - self._chunking_output = ( - start_line.method in ('POST', 'PUT', 'PATCH') and - 'Content-Length' not in headers and - 'Transfer-Encoding' not in headers) - else: - self._response_start_line = start_line - lines.append(utf8('HTTP/1.1 %d %s' % (start_line[1], start_line[2]))) - self._chunking_output = ( - # TODO: should this use - # self._request_start_line.version or - # start_line.version? - self._request_start_line.version == 'HTTP/1.1' and - # 1xx, 204 and 304 responses have no body (not even a zero-length - # body), and so should not have either Content-Length or - # Transfer-Encoding headers. - start_line.code not in (204, 304) and - (start_line.code < 100 or start_line.code >= 200) and - # No need to chunk the output if a Content-Length is specified. - 'Content-Length' not in headers and - # Applications are discouraged from touching Transfer-Encoding, - # but if they do, leave it alone. - 'Transfer-Encoding' not in headers) - # If connection to a 1.1 client will be closed, inform client - if (self._request_start_line.version == 'HTTP/1.1' and self._disconnect_on_finish): - headers['Connection'] = 'close' - # If a 1.0 client asked for keep-alive, add the header. - if (self._request_start_line.version == 'HTTP/1.0' and - self._request_headers.get('Connection', '').lower() == 'keep-alive'): - headers['Connection'] = 'Keep-Alive' - if self._chunking_output: - headers['Transfer-Encoding'] = 'chunked' - if (not self.is_client and - (self._request_start_line.method == 'HEAD' or - start_line.code == 304)): - self._expected_content_remaining = 0 - elif 'Content-Length' in headers: - self._expected_content_remaining = int(headers['Content-Length']) - else: - self._expected_content_remaining = None - # TODO: headers are supposed to be of type str, but we still have some - # cases that let bytes slip through. Remove these native_str calls when those - # are fixed. - header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) - if PY3: - lines.extend(l.encode('latin1') for l in header_lines) - else: - lines.extend(header_lines) - for line in lines: - if b'\n' in line: - raise ValueError('Newline in header: ' + repr(line)) - future = None - if self.stream.closed(): - future = self._write_future = Future() - future.set_exception(iostream.StreamClosedError()) - future.exception() - else: - if callback is not None: - warnings.warn("callback argument is deprecated, use returned Future instead", - DeprecationWarning) - self._write_callback = stack_context.wrap(callback) - else: - future = self._write_future = Future() - data = b"\r\n".join(lines) + b"\r\n\r\n" - if chunk: - data += self._format_chunk(chunk) - self._pending_write = self.stream.write(data) - future_add_done_callback(self._pending_write, self._on_write_complete) - return future - - def _format_chunk(self, chunk): - if self._expected_content_remaining is not None: - self._expected_content_remaining -= len(chunk) - if self._expected_content_remaining < 0: - # Close the stream now to stop further framing errors. - self.stream.close() - raise httputil.HTTPOutputError( - "Tried to write more data than Content-Length") - if self._chunking_output and chunk: - # Don't write out empty chunks because that means END-OF-STREAM - # with chunked encoding - return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n" - else: - return chunk - - def write(self, chunk, callback=None): - """Implements `.HTTPConnection.write`. - - For backwards compatibility it is allowed but deprecated to - skip `write_headers` and instead call `write()` with a - pre-encoded header block. - """ - future = None - if self.stream.closed(): - future = self._write_future = Future() - self._write_future.set_exception(iostream.StreamClosedError()) - self._write_future.exception() - else: - if callback is not None: - warnings.warn("callback argument is deprecated, use returned Future instead", - DeprecationWarning) - self._write_callback = stack_context.wrap(callback) - else: - future = self._write_future = Future() - self._pending_write = self.stream.write(self._format_chunk(chunk)) - self._pending_write.add_done_callback(self._on_write_complete) - return future - - def finish(self): - """Implements `.HTTPConnection.finish`.""" - if (self._expected_content_remaining is not None and - self._expected_content_remaining != 0 and - not self.stream.closed()): - self.stream.close() - raise httputil.HTTPOutputError( - "Tried to write %d bytes less than Content-Length" % - self._expected_content_remaining) - if self._chunking_output: - if not self.stream.closed(): - self._pending_write = self.stream.write(b"0\r\n\r\n") - self._pending_write.add_done_callback(self._on_write_complete) - self._write_finished = True - # If the app finished the request while we're still reading, - # divert any remaining data away from the delegate and - # close the connection when we're done sending our response. - # Closing the connection is the only way to avoid reading the - # whole input body. - if not self._read_finished: - self._disconnect_on_finish = True - # No more data is coming, so instruct TCP to send any remaining - # data immediately instead of waiting for a full packet or ack. - self.stream.set_nodelay(True) - if self._pending_write is None: - self._finish_request(None) - else: - future_add_done_callback(self._pending_write, self._finish_request) - - def _on_write_complete(self, future): - exc = future.exception() - if exc is not None and not isinstance(exc, iostream.StreamClosedError): - future.result() - if self._write_callback is not None: - callback = self._write_callback - self._write_callback = None - self.stream.io_loop.add_callback(callback) - if self._write_future is not None: - future = self._write_future - self._write_future = None - future_set_result_unless_cancelled(future, None) - - def _can_keep_alive(self, start_line, headers): - if self.params.no_keep_alive: - return False - connection_header = headers.get("Connection") - if connection_header is not None: - connection_header = connection_header.lower() - if start_line.version == "HTTP/1.1": - return connection_header != "close" - elif ("Content-Length" in headers or - headers.get("Transfer-Encoding", "").lower() == "chunked" or - getattr(start_line, 'method', None) in ("HEAD", "GET")): - # start_line may be a request or response start line; only - # the former has a method attribute. - return connection_header == "keep-alive" - return False - - def _finish_request(self, future): - self._clear_callbacks() - if not self.is_client and self._disconnect_on_finish: - self.close() - return - # Turn Nagle's algorithm back on, leaving the stream in its - # default state for the next request. - self.stream.set_nodelay(False) - if not self._finish_future.done(): - future_set_result_unless_cancelled(self._finish_future, None) - - def _parse_headers(self, data): - # The lstrip removes newlines that some implementations sometimes - # insert between messages of a reused connection. Per RFC 7230, - # we SHOULD ignore at least one empty line before the request. - # http://tools.ietf.org/html/rfc7230#section-3.5 - data = native_str(data.decode('latin1')).lstrip("\r\n") - # RFC 7230 section allows for both CRLF and bare LF. - eol = data.find("\n") - start_line = data[:eol].rstrip("\r") - headers = httputil.HTTPHeaders.parse(data[eol:]) - return start_line, headers - - def _read_body(self, code, headers, delegate): - if "Content-Length" in headers: - if "Transfer-Encoding" in headers: - # Response cannot contain both Content-Length and - # Transfer-Encoding headers. - # http://tools.ietf.org/html/rfc7230#section-3.3.3 - raise httputil.HTTPInputError( - "Response with both Transfer-Encoding and Content-Length") - if "," in headers["Content-Length"]: - # Proxies sometimes cause Content-Length headers to get - # duplicated. If all the values are identical then we can - # use them but if they differ it's an error. - pieces = re.split(r',\s*', headers["Content-Length"]) - if any(i != pieces[0] for i in pieces): - raise httputil.HTTPInputError( - "Multiple unequal Content-Lengths: %r" % - headers["Content-Length"]) - headers["Content-Length"] = pieces[0] - - try: - content_length = int(headers["Content-Length"]) - except ValueError: - # Handles non-integer Content-Length value. - raise httputil.HTTPInputError( - "Only integer Content-Length is allowed: %s" % headers["Content-Length"]) - - if content_length > self._max_body_size: - raise httputil.HTTPInputError("Content-Length too long") - else: - content_length = None - - if code == 204: - # This response code is not allowed to have a non-empty body, - # and has an implicit length of zero instead of read-until-close. - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 - if ("Transfer-Encoding" in headers or - content_length not in (None, 0)): - raise httputil.HTTPInputError( - "Response with code %d should not have body" % code) - content_length = 0 - - if content_length is not None: - return self._read_fixed_body(content_length, delegate) - if headers.get("Transfer-Encoding", "").lower() == "chunked": - return self._read_chunked_body(delegate) - if self.is_client: - return self._read_body_until_close(delegate) - return None - - @gen.coroutine - def _read_fixed_body(self, content_length, delegate): - while content_length > 0: - body = yield self.stream.read_bytes( - min(self.params.chunk_size, content_length), partial=True) - content_length -= len(body) - if not self._write_finished or self.is_client: - with _ExceptionLoggingContext(app_log): - ret = delegate.data_received(body) - if ret is not None: - yield ret - - @gen.coroutine - def _read_chunked_body(self, delegate): - # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 - total_size = 0 - while True: - chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) - chunk_len = int(chunk_len.strip(), 16) - if chunk_len == 0: - crlf = yield self.stream.read_bytes(2) - if crlf != b'\r\n': - raise httputil.HTTPInputError("improperly terminated chunked request") - return - total_size += chunk_len - if total_size > self._max_body_size: - raise httputil.HTTPInputError("chunked body too large") - bytes_to_read = chunk_len - while bytes_to_read: - chunk = yield self.stream.read_bytes( - min(bytes_to_read, self.params.chunk_size), partial=True) - bytes_to_read -= len(chunk) - if not self._write_finished or self.is_client: - with _ExceptionLoggingContext(app_log): - ret = delegate.data_received(chunk) - if ret is not None: - yield ret - # chunk ends with \r\n - crlf = yield self.stream.read_bytes(2) - assert crlf == b"\r\n" - - @gen.coroutine - def _read_body_until_close(self, delegate): - body = yield self.stream.read_until_close() - if not self._write_finished or self.is_client: - with _ExceptionLoggingContext(app_log): - delegate.data_received(body) - - -class _GzipMessageDelegate(httputil.HTTPMessageDelegate): - """Wraps an `HTTPMessageDelegate` to decode ``Content-Encoding: gzip``. - """ - def __init__(self, delegate, chunk_size): - self._delegate = delegate - self._chunk_size = chunk_size - self._decompressor = None - - def headers_received(self, start_line, headers): - if headers.get("Content-Encoding") == "gzip": - self._decompressor = GzipDecompressor() - # Downstream delegates will only see uncompressed data, - # so rename the content-encoding header. - # (but note that curl_httpclient doesn't do this). - headers.add("X-Consumed-Content-Encoding", - headers["Content-Encoding"]) - del headers["Content-Encoding"] - return self._delegate.headers_received(start_line, headers) - - @gen.coroutine - def data_received(self, chunk): - if self._decompressor: - compressed_data = chunk - while compressed_data: - decompressed = self._decompressor.decompress( - compressed_data, self._chunk_size) - if decompressed: - ret = self._delegate.data_received(decompressed) - if ret is not None: - yield ret - compressed_data = self._decompressor.unconsumed_tail - else: - ret = self._delegate.data_received(chunk) - if ret is not None: - yield ret - - def finish(self): - if self._decompressor is not None: - tail = self._decompressor.flush() - if tail: - # I believe the tail will always be empty (i.e. - # decompress will return all it can). The purpose - # of the flush call is to detect errors such - # as truncated input. But in case it ever returns - # anything, treat it as an extra chunk - self._delegate.data_received(tail) - return self._delegate.finish() - - def on_connection_close(self): - return self._delegate.on_connection_close() - - -class HTTP1ServerConnection(object): - """An HTTP/1.x server.""" - def __init__(self, stream, params=None, context=None): - """ - :arg stream: an `.IOStream` - :arg params: a `.HTTP1ConnectionParameters` or None - :arg context: an opaque application-defined object that is accessible - as ``connection.context`` - """ - self.stream = stream - if params is None: - params = HTTP1ConnectionParameters() - self.params = params - self.context = context - self._serving_future = None - - @gen.coroutine - def close(self): - """Closes the connection. - - Returns a `.Future` that resolves after the serving loop has exited. - """ - self.stream.close() - # Block until the serving loop is done, but ignore any exceptions - # (start_serving is already responsible for logging them). - try: - yield self._serving_future - except Exception: - pass - - def start_serving(self, delegate): - """Starts serving requests on this connection. - - :arg delegate: a `.HTTPServerConnectionDelegate` - """ - assert isinstance(delegate, httputil.HTTPServerConnectionDelegate) - self._serving_future = self._server_request_loop(delegate) - # Register the future on the IOLoop so its errors get logged. - self.stream.io_loop.add_future(self._serving_future, - lambda f: f.result()) - - @gen.coroutine - def _server_request_loop(self, delegate): - try: - while True: - conn = HTTP1Connection(self.stream, False, - self.params, self.context) - request_delegate = delegate.start_request(self, conn) - try: - ret = yield conn.read_response(request_delegate) - except (iostream.StreamClosedError, - iostream.UnsatisfiableReadError): - return - except _QuietException: - # This exception was already logged. - conn.close() - return - except Exception: - gen_log.error("Uncaught exception", exc_info=True) - conn.close() - return - if not ret: - return - yield gen.moment - finally: - delegate.on_close(self) diff --git a/lib/tornado/httpclient.py b/lib/tornado/httpclient.py deleted file mode 100755 index 5ed2ee67..00000000 --- a/lib/tornado/httpclient.py +++ /dev/null @@ -1,748 +0,0 @@ -"""Blocking and non-blocking HTTP client interfaces. - -This module defines a common interface shared by two implementations, -``simple_httpclient`` and ``curl_httpclient``. Applications may either -instantiate their chosen implementation class directly or use the -`AsyncHTTPClient` class from this module, which selects an implementation -that can be overridden with the `AsyncHTTPClient.configure` method. - -The default implementation is ``simple_httpclient``, and this is expected -to be suitable for most users' needs. However, some applications may wish -to switch to ``curl_httpclient`` for reasons such as the following: - -* ``curl_httpclient`` has some features not found in ``simple_httpclient``, - including support for HTTP proxies and the ability to use a specified - network interface. - -* ``curl_httpclient`` is more likely to be compatible with sites that are - not-quite-compliant with the HTTP spec, or sites that use little-exercised - features of HTTP. - -* ``curl_httpclient`` is faster. - -* ``curl_httpclient`` was the default prior to Tornado 2.0. - -Note that if you are using ``curl_httpclient``, it is highly -recommended that you use a recent version of ``libcurl`` and -``pycurl``. Currently the minimum supported version of libcurl is -7.22.0, and the minimum version of pycurl is 7.18.2. It is highly -recommended that your ``libcurl`` installation is built with -asynchronous DNS resolver (threaded or c-ares), otherwise you may -encounter various problems with request timeouts (for more -information, see -http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS -and comments in curl_httpclient.py). - -To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup:: - - AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") -""" - -from __future__ import absolute_import, division, print_function - -import functools -import time -import warnings -import weakref - -from tornado.concurrent import Future, future_set_result_unless_cancelled -from tornado.escape import utf8, native_str -from tornado import gen, httputil, stack_context -from tornado.ioloop import IOLoop -from tornado.util import Configurable - - -class HTTPClient(object): - """A blocking HTTP client. - - This interface is provided to make it easier to share code between - synchronous and asynchronous applications. Applications that are - running an `.IOLoop` must use `AsyncHTTPClient` instead. - - Typical usage looks like this:: - - http_client = httpclient.HTTPClient() - try: - response = http_client.fetch("http://www.google.com/") - print(response.body) - except httpclient.HTTPError as e: - # HTTPError is raised for non-200 responses; the response - # can be found in e.response. - print("Error: " + str(e)) - except Exception as e: - # Other errors are possible, such as IOError. - print("Error: " + str(e)) - http_client.close() - - .. versionchanged:: 5.0 - - Due to limitations in `asyncio`, it is no longer possible to - use the synchronous ``HTTPClient`` while an `.IOLoop` is running. - Use `AsyncHTTPClient` instead. - - """ - def __init__(self, async_client_class=None, **kwargs): - # Initialize self._closed at the beginning of the constructor - # so that an exception raised here doesn't lead to confusing - # failures in __del__. - self._closed = True - self._io_loop = IOLoop(make_current=False) - if async_client_class is None: - async_client_class = AsyncHTTPClient - # Create the client while our IOLoop is "current", without - # clobbering the thread's real current IOLoop (if any). - self._async_client = self._io_loop.run_sync( - gen.coroutine(lambda: async_client_class(**kwargs))) - self._closed = False - - def __del__(self): - self.close() - - def close(self): - """Closes the HTTPClient, freeing any resources used.""" - if not self._closed: - self._async_client.close() - self._io_loop.close() - self._closed = True - - def fetch(self, request, **kwargs): - """Executes a request, returning an `HTTPResponse`. - - The request may be either a string URL or an `HTTPRequest` object. - If it is a string, we construct an `HTTPRequest` using any additional - kwargs: ``HTTPRequest(request, **kwargs)`` - - If an error occurs during the fetch, we raise an `HTTPError` unless - the ``raise_error`` keyword argument is set to False. - """ - response = self._io_loop.run_sync(functools.partial( - self._async_client.fetch, request, **kwargs)) - return response - - -class AsyncHTTPClient(Configurable): - """An non-blocking HTTP client. - - Example usage:: - - async def f(): - http_client = AsyncHTTPClient() - try: - response = await http_client.fetch("http://www.google.com") - except Exception as e: - print("Error: %s" % e) - else: - print(response.body) - - The constructor for this class is magic in several respects: It - actually creates an instance of an implementation-specific - subclass, and instances are reused as a kind of pseudo-singleton - (one per `.IOLoop`). The keyword argument ``force_instance=True`` - can be used to suppress this singleton behavior. Unless - ``force_instance=True`` is used, no arguments should be passed to - the `AsyncHTTPClient` constructor. The implementation subclass as - well as arguments to its constructor can be set with the static - method `configure()` - - All `AsyncHTTPClient` implementations support a ``defaults`` - keyword argument, which can be used to set default values for - `HTTPRequest` attributes. For example:: - - AsyncHTTPClient.configure( - None, defaults=dict(user_agent="MyUserAgent")) - # or with force_instance: - client = AsyncHTTPClient(force_instance=True, - defaults=dict(user_agent="MyUserAgent")) - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - """ - @classmethod - def configurable_base(cls): - return AsyncHTTPClient - - @classmethod - def configurable_default(cls): - from tornado.simple_httpclient import SimpleAsyncHTTPClient - return SimpleAsyncHTTPClient - - @classmethod - def _async_clients(cls): - attr_name = '_async_client_dict_' + cls.__name__ - if not hasattr(cls, attr_name): - setattr(cls, attr_name, weakref.WeakKeyDictionary()) - return getattr(cls, attr_name) - - def __new__(cls, force_instance=False, **kwargs): - io_loop = IOLoop.current() - if force_instance: - instance_cache = None - else: - instance_cache = cls._async_clients() - if instance_cache is not None and io_loop in instance_cache: - return instance_cache[io_loop] - instance = super(AsyncHTTPClient, cls).__new__(cls, **kwargs) - # Make sure the instance knows which cache to remove itself from. - # It can't simply call _async_clients() because we may be in - # __new__(AsyncHTTPClient) but instance.__class__ may be - # SimpleAsyncHTTPClient. - instance._instance_cache = instance_cache - if instance_cache is not None: - instance_cache[instance.io_loop] = instance - return instance - - def initialize(self, defaults=None): - self.io_loop = IOLoop.current() - self.defaults = dict(HTTPRequest._DEFAULTS) - if defaults is not None: - self.defaults.update(defaults) - self._closed = False - - def close(self): - """Destroys this HTTP client, freeing any file descriptors used. - - This method is **not needed in normal use** due to the way - that `AsyncHTTPClient` objects are transparently reused. - ``close()`` is generally only necessary when either the - `.IOLoop` is also being closed, or the ``force_instance=True`` - argument was used when creating the `AsyncHTTPClient`. - - No other methods may be called on the `AsyncHTTPClient` after - ``close()``. - - """ - if self._closed: - return - self._closed = True - if self._instance_cache is not None: - if self._instance_cache.get(self.io_loop) is not self: - raise RuntimeError("inconsistent AsyncHTTPClient cache") - del self._instance_cache[self.io_loop] - - def fetch(self, request, callback=None, raise_error=True, **kwargs): - """Executes a request, asynchronously returning an `HTTPResponse`. - - The request may be either a string URL or an `HTTPRequest` object. - If it is a string, we construct an `HTTPRequest` using any additional - kwargs: ``HTTPRequest(request, **kwargs)`` - - This method returns a `.Future` whose result is an - `HTTPResponse`. By default, the ``Future`` will raise an - `HTTPError` if the request returned a non-200 response code - (other errors may also be raised if the server could not be - contacted). Instead, if ``raise_error`` is set to False, the - response will always be returned regardless of the response - code. - - If a ``callback`` is given, it will be invoked with the `HTTPResponse`. - In the callback interface, `HTTPError` is not automatically raised. - Instead, you must check the response's ``error`` attribute or - call its `~HTTPResponse.rethrow` method. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in 6.0. Use the returned `.Future` instead. - - The ``raise_error=False`` argument currently suppresses - *all* errors, encapsulating them in `HTTPResponse` objects - with a 599 response code. This will change in Tornado 6.0: - ``raise_error=False`` will only affect the `HTTPError` - raised when a non-200 response code is used. - - """ - if self._closed: - raise RuntimeError("fetch() called on closed AsyncHTTPClient") - if not isinstance(request, HTTPRequest): - request = HTTPRequest(url=request, **kwargs) - else: - if kwargs: - raise ValueError("kwargs can't be used if request is an HTTPRequest object") - # We may modify this (to add Host, Accept-Encoding, etc), - # so make sure we don't modify the caller's object. This is also - # where normal dicts get converted to HTTPHeaders objects. - request.headers = httputil.HTTPHeaders(request.headers) - request = _RequestProxy(request, self.defaults) - future = Future() - if callback is not None: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - callback = stack_context.wrap(callback) - - def handle_future(future): - exc = future.exception() - if isinstance(exc, HTTPError) and exc.response is not None: - response = exc.response - elif exc is not None: - response = HTTPResponse( - request, 599, error=exc, - request_time=time.time() - request.start_time) - else: - response = future.result() - self.io_loop.add_callback(callback, response) - future.add_done_callback(handle_future) - - def handle_response(response): - if raise_error and response.error: - if isinstance(response.error, HTTPError): - response.error.response = response - future.set_exception(response.error) - else: - if response.error and not response._error_is_response_code: - warnings.warn("raise_error=False will allow '%s' to be raised in the future" % - response.error, DeprecationWarning) - future_set_result_unless_cancelled(future, response) - self.fetch_impl(request, handle_response) - return future - - def fetch_impl(self, request, callback): - raise NotImplementedError() - - @classmethod - def configure(cls, impl, **kwargs): - """Configures the `AsyncHTTPClient` subclass to use. - - ``AsyncHTTPClient()`` actually creates an instance of a subclass. - This method may be called with either a class object or the - fully-qualified name of such a class (or ``None`` to use the default, - ``SimpleAsyncHTTPClient``) - - If additional keyword arguments are given, they will be passed - to the constructor of each subclass instance created. The - keyword argument ``max_clients`` determines the maximum number - of simultaneous `~AsyncHTTPClient.fetch()` operations that can - execute in parallel on each `.IOLoop`. Additional arguments - may be supported depending on the implementation class in use. - - Example:: - - AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") - """ - super(AsyncHTTPClient, cls).configure(impl, **kwargs) - - -class HTTPRequest(object): - """HTTP client request object.""" - - # Default values for HTTPRequest parameters. - # Merged with the values on the request object by AsyncHTTPClient - # implementations. - _DEFAULTS = dict( - connect_timeout=20.0, - request_timeout=20.0, - follow_redirects=True, - max_redirects=5, - decompress_response=True, - proxy_password='', - allow_nonstandard_methods=False, - validate_cert=True) - - def __init__(self, url, method="GET", headers=None, body=None, - auth_username=None, auth_password=None, auth_mode=None, - connect_timeout=None, request_timeout=None, - if_modified_since=None, follow_redirects=None, - max_redirects=None, user_agent=None, use_gzip=None, - network_interface=None, streaming_callback=None, - header_callback=None, prepare_curl_callback=None, - proxy_host=None, proxy_port=None, proxy_username=None, - proxy_password=None, proxy_auth_mode=None, - allow_nonstandard_methods=None, validate_cert=None, - ca_certs=None, allow_ipv6=None, client_key=None, - client_cert=None, body_producer=None, - expect_100_continue=False, decompress_response=None, - ssl_options=None): - r"""All parameters except ``url`` are optional. - - :arg str url: URL to fetch - :arg str method: HTTP method, e.g. "GET" or "POST" - :arg headers: Additional HTTP headers to pass on the request - :type headers: `~tornado.httputil.HTTPHeaders` or `dict` - :arg body: HTTP request body as a string (byte or unicode; if unicode - the utf-8 encoding will be used) - :arg body_producer: Callable used for lazy/asynchronous request bodies. - It is called with one argument, a ``write`` function, and should - return a `.Future`. It should call the write function with new - data as it becomes available. The write function returns a - `.Future` which can be used for flow control. - Only one of ``body`` and ``body_producer`` may - be specified. ``body_producer`` is not supported on - ``curl_httpclient``. When using ``body_producer`` it is recommended - to pass a ``Content-Length`` in the headers as otherwise chunked - encoding will be used, and many servers do not support chunked - encoding on requests. New in Tornado 4.0 - :arg str auth_username: Username for HTTP authentication - :arg str auth_password: Password for HTTP authentication - :arg str auth_mode: Authentication mode; default is "basic". - Allowed values are implementation-defined; ``curl_httpclient`` - supports "basic" and "digest"; ``simple_httpclient`` only supports - "basic" - :arg float connect_timeout: Timeout for initial connection in seconds, - default 20 seconds - :arg float request_timeout: Timeout for entire request in seconds, - default 20 seconds - :arg if_modified_since: Timestamp for ``If-Modified-Since`` header - :type if_modified_since: `datetime` or `float` - :arg bool follow_redirects: Should redirects be followed automatically - or return the 3xx response? Default True. - :arg int max_redirects: Limit for ``follow_redirects``, default 5. - :arg str user_agent: String to send as ``User-Agent`` header - :arg bool decompress_response: Request a compressed response from - the server and decompress it after downloading. Default is True. - New in Tornado 4.0. - :arg bool use_gzip: Deprecated alias for ``decompress_response`` - since Tornado 4.0. - :arg str network_interface: Network interface to use for request. - ``curl_httpclient`` only; see note below. - :arg collections.abc.Callable streaming_callback: If set, ``streaming_callback`` will - be run with each chunk of data as it is received, and - ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in - the final response. - :arg collections.abc.Callable header_callback: If set, ``header_callback`` will - be run with each header line as it is received (including the - first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line - containing only ``\r\n``. All lines include the trailing newline - characters). ``HTTPResponse.headers`` will be empty in the final - response. This is most useful in conjunction with - ``streaming_callback``, because it's the only way to get access to - header data while the request is in progress. - :arg collections.abc.Callable prepare_curl_callback: If set, will be called with - a ``pycurl.Curl`` object to allow the application to make additional - ``setopt`` calls. - :arg str proxy_host: HTTP proxy hostname. To use proxies, - ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username``, - ``proxy_pass`` and ``proxy_auth_mode`` are optional. Proxies are - currently only supported with ``curl_httpclient``. - :arg int proxy_port: HTTP proxy port - :arg str proxy_username: HTTP proxy username - :arg str proxy_password: HTTP proxy password - :arg str proxy_auth_mode: HTTP proxy Authentication mode; - default is "basic". supports "basic" and "digest" - :arg bool allow_nonstandard_methods: Allow unknown values for ``method`` - argument? Default is False. - :arg bool validate_cert: For HTTPS requests, validate the server's - certificate? Default is True. - :arg str ca_certs: filename of CA certificates in PEM format, - or None to use defaults. See note below when used with - ``curl_httpclient``. - :arg str client_key: Filename for client SSL key, if any. See - note below when used with ``curl_httpclient``. - :arg str client_cert: Filename for client SSL certificate, if any. - See note below when used with ``curl_httpclient``. - :arg ssl.SSLContext ssl_options: `ssl.SSLContext` object for use in - ``simple_httpclient`` (unsupported by ``curl_httpclient``). - Overrides ``validate_cert``, ``ca_certs``, ``client_key``, - and ``client_cert``. - :arg bool allow_ipv6: Use IPv6 when available? Default is true. - :arg bool expect_100_continue: If true, send the - ``Expect: 100-continue`` header and wait for a continue response - before sending the request body. Only supported with - simple_httpclient. - - .. note:: - - When using ``curl_httpclient`` certain options may be - inherited by subsequent fetches because ``pycurl`` does - not allow them to be cleanly reset. This applies to the - ``ca_certs``, ``client_key``, ``client_cert``, and - ``network_interface`` arguments. If you use these - options, you should pass them on every request (you don't - have to always use the same values, but it's not possible - to mix requests that specify these options with ones that - use the defaults). - - .. versionadded:: 3.1 - The ``auth_mode`` argument. - - .. versionadded:: 4.0 - The ``body_producer`` and ``expect_100_continue`` arguments. - - .. versionadded:: 4.2 - The ``ssl_options`` argument. - - .. versionadded:: 4.5 - The ``proxy_auth_mode`` argument. - """ - # Note that some of these attributes go through property setters - # defined below. - self.headers = headers - if if_modified_since: - self.headers["If-Modified-Since"] = httputil.format_timestamp( - if_modified_since) - self.proxy_host = proxy_host - self.proxy_port = proxy_port - self.proxy_username = proxy_username - self.proxy_password = proxy_password - self.proxy_auth_mode = proxy_auth_mode - self.url = url - self.method = method - self.body = body - self.body_producer = body_producer - self.auth_username = auth_username - self.auth_password = auth_password - self.auth_mode = auth_mode - self.connect_timeout = connect_timeout - self.request_timeout = request_timeout - self.follow_redirects = follow_redirects - self.max_redirects = max_redirects - self.user_agent = user_agent - if decompress_response is not None: - self.decompress_response = decompress_response - else: - self.decompress_response = use_gzip - self.network_interface = network_interface - self.streaming_callback = streaming_callback - self.header_callback = header_callback - self.prepare_curl_callback = prepare_curl_callback - self.allow_nonstandard_methods = allow_nonstandard_methods - self.validate_cert = validate_cert - self.ca_certs = ca_certs - self.allow_ipv6 = allow_ipv6 - self.client_key = client_key - self.client_cert = client_cert - self.ssl_options = ssl_options - self.expect_100_continue = expect_100_continue - self.start_time = time.time() - - @property - def headers(self): - return self._headers - - @headers.setter - def headers(self, value): - if value is None: - self._headers = httputil.HTTPHeaders() - else: - self._headers = value - - @property - def body(self): - return self._body - - @body.setter - def body(self, value): - self._body = utf8(value) - - @property - def body_producer(self): - return self._body_producer - - @body_producer.setter - def body_producer(self, value): - self._body_producer = stack_context.wrap(value) - - @property - def streaming_callback(self): - return self._streaming_callback - - @streaming_callback.setter - def streaming_callback(self, value): - self._streaming_callback = stack_context.wrap(value) - - @property - def header_callback(self): - return self._header_callback - - @header_callback.setter - def header_callback(self, value): - self._header_callback = stack_context.wrap(value) - - @property - def prepare_curl_callback(self): - return self._prepare_curl_callback - - @prepare_curl_callback.setter - def prepare_curl_callback(self, value): - self._prepare_curl_callback = stack_context.wrap(value) - - -class HTTPResponse(object): - """HTTP Response object. - - Attributes: - - * request: HTTPRequest object - - * code: numeric HTTP status code, e.g. 200 or 404 - - * reason: human-readable reason phrase describing the status code - - * headers: `tornado.httputil.HTTPHeaders` object - - * effective_url: final location of the resource after following any - redirects - - * buffer: ``cStringIO`` object for response body - - * body: response body as bytes (created on demand from ``self.buffer``) - - * error: Exception object, if any - - * request_time: seconds from request start to finish. Includes all network - operations from DNS resolution to receiving the last byte of data. - Does not include time spent in the queue (due to the ``max_clients`` option). - If redirects were followed, only includes the final request. - - * start_time: Time at which the HTTP operation started, based on `time.time` - (not the monotonic clock used by `.IOLoop.time`). May be ``None`` if the request - timed out while in the queue. - - * time_info: dictionary of diagnostic timing information from the request. - Available data are subject to change, but currently uses timings - available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, - plus ``queue``, which is the delay (if any) introduced by waiting for - a slot under `AsyncHTTPClient`'s ``max_clients`` setting. - - .. versionadded:: 5.1 - - Added the ``start_time`` attribute. - - .. versionchanged:: 5.1 - - The ``request_time`` attribute previously included time spent in the queue - for ``simple_httpclient``, but not in ``curl_httpclient``. Now queueing time - is excluded in both implementations. ``request_time`` is now more accurate for - ``curl_httpclient`` because it uses a monotonic clock when available. - """ - def __init__(self, request, code, headers=None, buffer=None, - effective_url=None, error=None, request_time=None, - time_info=None, reason=None, start_time=None): - if isinstance(request, _RequestProxy): - self.request = request.request - else: - self.request = request - self.code = code - self.reason = reason or httputil.responses.get(code, "Unknown") - if headers is not None: - self.headers = headers - else: - self.headers = httputil.HTTPHeaders() - self.buffer = buffer - self._body = None - if effective_url is None: - self.effective_url = request.url - else: - self.effective_url = effective_url - self._error_is_response_code = False - if error is None: - if self.code < 200 or self.code >= 300: - self._error_is_response_code = True - self.error = HTTPError(self.code, message=self.reason, - response=self) - else: - self.error = None - else: - self.error = error - self.start_time = start_time - self.request_time = request_time - self.time_info = time_info or {} - - @property - def body(self): - if self.buffer is None: - return None - elif self._body is None: - self._body = self.buffer.getvalue() - - return self._body - - def rethrow(self): - """If there was an error on the request, raise an `HTTPError`.""" - if self.error: - raise self.error - - def __repr__(self): - args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items())) - return "%s(%s)" % (self.__class__.__name__, args) - - -class HTTPClientError(Exception): - """Exception thrown for an unsuccessful HTTP request. - - Attributes: - - * ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is - used when no HTTP response was received, e.g. for a timeout. - - * ``response`` - `HTTPResponse` object, if any. - - Note that if ``follow_redirects`` is False, redirects become HTTPErrors, - and you can look at ``error.response.headers['Location']`` to see the - destination of the redirect. - - .. versionchanged:: 5.1 - - Renamed from ``HTTPError`` to ``HTTPClientError`` to avoid collisions with - `tornado.web.HTTPError`. The name ``tornado.httpclient.HTTPError`` remains - as an alias. - """ - def __init__(self, code, message=None, response=None): - self.code = code - self.message = message or httputil.responses.get(code, "Unknown") - self.response = response - super(HTTPClientError, self).__init__(code, message, response) - - def __str__(self): - return "HTTP %d: %s" % (self.code, self.message) - - # There is a cyclic reference between self and self.response, - # which breaks the default __repr__ implementation. - # (especially on pypy, which doesn't have the same recursion - # detection as cpython). - __repr__ = __str__ - - -HTTPError = HTTPClientError - - -class _RequestProxy(object): - """Combines an object with a dictionary of defaults. - - Used internally by AsyncHTTPClient implementations. - """ - def __init__(self, request, defaults): - self.request = request - self.defaults = defaults - - def __getattr__(self, name): - request_attr = getattr(self.request, name) - if request_attr is not None: - return request_attr - elif self.defaults is not None: - return self.defaults.get(name, None) - else: - return None - - -def main(): - from tornado.options import define, options, parse_command_line - define("print_headers", type=bool, default=False) - define("print_body", type=bool, default=True) - define("follow_redirects", type=bool, default=True) - define("validate_cert", type=bool, default=True) - define("proxy_host", type=str) - define("proxy_port", type=int) - args = parse_command_line() - client = HTTPClient() - for arg in args: - try: - response = client.fetch(arg, - follow_redirects=options.follow_redirects, - validate_cert=options.validate_cert, - proxy_host=options.proxy_host, - proxy_port=options.proxy_port, - ) - except HTTPError as e: - if e.response is not None: - response = e.response - else: - raise - if options.print_headers: - print(response.headers) - if options.print_body: - print(native_str(response.body)) - client.close() - - -if __name__ == "__main__": - main() diff --git a/lib/tornado/httpserver.py b/lib/tornado/httpserver.py deleted file mode 100755 index 3498d71f..00000000 --- a/lib/tornado/httpserver.py +++ /dev/null @@ -1,330 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A non-blocking, single-threaded HTTP server. - -Typical applications have little direct interaction with the `HTTPServer` -class except to start a server at the beginning of the process -(and even that is often done indirectly via `tornado.web.Application.listen`). - -.. versionchanged:: 4.0 - - The ``HTTPRequest`` class that used to live in this module has been moved - to `tornado.httputil.HTTPServerRequest`. The old name remains as an alias. -""" - -from __future__ import absolute_import, division, print_function - -import socket - -from tornado.escape import native_str -from tornado.http1connection import HTTP1ServerConnection, HTTP1ConnectionParameters -from tornado import gen -from tornado import httputil -from tornado import iostream -from tornado import netutil -from tornado.tcpserver import TCPServer -from tornado.util import Configurable - - -class HTTPServer(TCPServer, Configurable, - httputil.HTTPServerConnectionDelegate): - r"""A non-blocking, single-threaded HTTP server. - - A server is defined by a subclass of `.HTTPServerConnectionDelegate`, - or, for backwards compatibility, a callback that takes an - `.HTTPServerRequest` as an argument. The delegate is usually a - `tornado.web.Application`. - - `HTTPServer` supports keep-alive connections by default - (automatically for HTTP/1.1, or for HTTP/1.0 when the client - requests ``Connection: keep-alive``). - - If ``xheaders`` is ``True``, we support the - ``X-Real-Ip``/``X-Forwarded-For`` and - ``X-Scheme``/``X-Forwarded-Proto`` headers, which override the - remote IP and URI scheme/protocol for all requests. These headers - are useful when running Tornado behind a reverse proxy or load - balancer. The ``protocol`` argument can also be set to ``https`` - if Tornado is run behind an SSL-decoding proxy that does not set one of - the supported ``xheaders``. - - By default, when parsing the ``X-Forwarded-For`` header, Tornado will - select the last (i.e., the closest) address on the list of hosts as the - remote host IP address. To select the next server in the chain, a list of - trusted downstream hosts may be passed as the ``trusted_downstream`` - argument. These hosts will be skipped when parsing the ``X-Forwarded-For`` - header. - - To make this server serve SSL traffic, send the ``ssl_options`` keyword - argument with an `ssl.SSLContext` object. For compatibility with older - versions of Python ``ssl_options`` may also be a dictionary of keyword - arguments for the `ssl.wrap_socket` method.:: - - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"), - os.path.join(data_dir, "mydomain.key")) - HTTPServer(application, ssl_options=ssl_ctx) - - `HTTPServer` initialization follows one of three patterns (the - initialization methods are defined on `tornado.tcpserver.TCPServer`): - - 1. `~tornado.tcpserver.TCPServer.listen`: simple single-process:: - - server = HTTPServer(app) - server.listen(8888) - IOLoop.current().start() - - In many cases, `tornado.web.Application.listen` can be used to avoid - the need to explicitly create the `HTTPServer`. - - 2. `~tornado.tcpserver.TCPServer.bind`/`~tornado.tcpserver.TCPServer.start`: - simple multi-process:: - - server = HTTPServer(app) - server.bind(8888) - server.start(0) # Forks multiple sub-processes - IOLoop.current().start() - - When using this interface, an `.IOLoop` must *not* be passed - to the `HTTPServer` constructor. `~.TCPServer.start` will always start - the server on the default singleton `.IOLoop`. - - 3. `~tornado.tcpserver.TCPServer.add_sockets`: advanced multi-process:: - - sockets = tornado.netutil.bind_sockets(8888) - tornado.process.fork_processes(0) - server = HTTPServer(app) - server.add_sockets(sockets) - IOLoop.current().start() - - The `~.TCPServer.add_sockets` interface is more complicated, - but it can be used with `tornado.process.fork_processes` to - give you more flexibility in when the fork happens. - `~.TCPServer.add_sockets` can also be used in single-process - servers if you want to create your listening sockets in some - way other than `tornado.netutil.bind_sockets`. - - .. versionchanged:: 4.0 - Added ``decompress_request``, ``chunk_size``, ``max_header_size``, - ``idle_connection_timeout``, ``body_timeout``, ``max_body_size`` - arguments. Added support for `.HTTPServerConnectionDelegate` - instances as ``request_callback``. - - .. versionchanged:: 4.1 - `.HTTPServerConnectionDelegate.start_request` is now called with - two arguments ``(server_conn, request_conn)`` (in accordance with the - documentation) instead of one ``(request_conn)``. - - .. versionchanged:: 4.2 - `HTTPServer` is now a subclass of `tornado.util.Configurable`. - - .. versionchanged:: 4.5 - Added the ``trusted_downstream`` argument. - - .. versionchanged:: 5.0 - The ``io_loop`` argument has been removed. - """ - def __init__(self, *args, **kwargs): - # Ignore args to __init__; real initialization belongs in - # initialize since we're Configurable. (there's something - # weird in initialization order between this class, - # Configurable, and TCPServer so we can't leave __init__ out - # completely) - pass - - def initialize(self, request_callback, no_keep_alive=False, - xheaders=False, ssl_options=None, protocol=None, - decompress_request=False, - chunk_size=None, max_header_size=None, - idle_connection_timeout=None, body_timeout=None, - max_body_size=None, max_buffer_size=None, - trusted_downstream=None): - self.request_callback = request_callback - self.xheaders = xheaders - self.protocol = protocol - self.conn_params = HTTP1ConnectionParameters( - decompress=decompress_request, - chunk_size=chunk_size, - max_header_size=max_header_size, - header_timeout=idle_connection_timeout or 3600, - max_body_size=max_body_size, - body_timeout=body_timeout, - no_keep_alive=no_keep_alive) - TCPServer.__init__(self, ssl_options=ssl_options, - max_buffer_size=max_buffer_size, - read_chunk_size=chunk_size) - self._connections = set() - self.trusted_downstream = trusted_downstream - - @classmethod - def configurable_base(cls): - return HTTPServer - - @classmethod - def configurable_default(cls): - return HTTPServer - - @gen.coroutine - def close_all_connections(self): - while self._connections: - # Peek at an arbitrary element of the set - conn = next(iter(self._connections)) - yield conn.close() - - def handle_stream(self, stream, address): - context = _HTTPRequestContext(stream, address, - self.protocol, - self.trusted_downstream) - conn = HTTP1ServerConnection( - stream, self.conn_params, context) - self._connections.add(conn) - conn.start_serving(self) - - def start_request(self, server_conn, request_conn): - if isinstance(self.request_callback, httputil.HTTPServerConnectionDelegate): - delegate = self.request_callback.start_request(server_conn, request_conn) - else: - delegate = _CallableAdapter(self.request_callback, request_conn) - - if self.xheaders: - delegate = _ProxyAdapter(delegate, request_conn) - - return delegate - - def on_close(self, server_conn): - self._connections.remove(server_conn) - - -class _CallableAdapter(httputil.HTTPMessageDelegate): - def __init__(self, request_callback, request_conn): - self.connection = request_conn - self.request_callback = request_callback - self.request = None - self.delegate = None - self._chunks = [] - - def headers_received(self, start_line, headers): - self.request = httputil.HTTPServerRequest( - connection=self.connection, start_line=start_line, - headers=headers) - - def data_received(self, chunk): - self._chunks.append(chunk) - - def finish(self): - self.request.body = b''.join(self._chunks) - self.request._parse_body() - self.request_callback(self.request) - - def on_connection_close(self): - self._chunks = None - - -class _HTTPRequestContext(object): - def __init__(self, stream, address, protocol, trusted_downstream=None): - self.address = address - # Save the socket's address family now so we know how to - # interpret self.address even after the stream is closed - # and its socket attribute replaced with None. - if stream.socket is not None: - self.address_family = stream.socket.family - else: - self.address_family = None - # In HTTPServerRequest we want an IP, not a full socket address. - if (self.address_family in (socket.AF_INET, socket.AF_INET6) and - address is not None): - self.remote_ip = address[0] - else: - # Unix (or other) socket; fake the remote address. - self.remote_ip = '0.0.0.0' - if protocol: - self.protocol = protocol - elif isinstance(stream, iostream.SSLIOStream): - self.protocol = "https" - else: - self.protocol = "http" - self._orig_remote_ip = self.remote_ip - self._orig_protocol = self.protocol - self.trusted_downstream = set(trusted_downstream or []) - - def __str__(self): - if self.address_family in (socket.AF_INET, socket.AF_INET6): - return self.remote_ip - elif isinstance(self.address, bytes): - # Python 3 with the -bb option warns about str(bytes), - # so convert it explicitly. - # Unix socket addresses are str on mac but bytes on linux. - return native_str(self.address) - else: - return str(self.address) - - def _apply_xheaders(self, headers): - """Rewrite the ``remote_ip`` and ``protocol`` fields.""" - # Squid uses X-Forwarded-For, others use X-Real-Ip - ip = headers.get("X-Forwarded-For", self.remote_ip) - # Skip trusted downstream hosts in X-Forwarded-For list - for ip in (cand.strip() for cand in reversed(ip.split(','))): - if ip not in self.trusted_downstream: - break - ip = headers.get("X-Real-Ip", ip) - if netutil.is_valid_ip(ip): - self.remote_ip = ip - # AWS uses X-Forwarded-Proto - proto_header = headers.get( - "X-Scheme", headers.get("X-Forwarded-Proto", - self.protocol)) - if proto_header: - # use only the last proto entry if there is more than one - # TODO: support trusting mutiple layers of proxied protocol - proto_header = proto_header.split(',')[-1].strip() - if proto_header in ("http", "https"): - self.protocol = proto_header - - def _unapply_xheaders(self): - """Undo changes from `_apply_xheaders`. - - Xheaders are per-request so they should not leak to the next - request on the same connection. - """ - self.remote_ip = self._orig_remote_ip - self.protocol = self._orig_protocol - - -class _ProxyAdapter(httputil.HTTPMessageDelegate): - def __init__(self, delegate, request_conn): - self.connection = request_conn - self.delegate = delegate - - def headers_received(self, start_line, headers): - self.connection.context._apply_xheaders(headers) - return self.delegate.headers_received(start_line, headers) - - def data_received(self, chunk): - return self.delegate.data_received(chunk) - - def finish(self): - self.delegate.finish() - self._cleanup() - - def on_connection_close(self): - self.delegate.on_connection_close() - self._cleanup() - - def _cleanup(self): - self.connection.context._unapply_xheaders() - - -HTTPRequest = httputil.HTTPServerRequest diff --git a/lib/tornado/httputil.py b/lib/tornado/httputil.py deleted file mode 100755 index 39614466..00000000 --- a/lib/tornado/httputil.py +++ /dev/null @@ -1,1095 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""HTTP utility code shared by clients and servers. - -This module also defines the `HTTPServerRequest` class which is exposed -via `tornado.web.RequestHandler.request`. -""" - -from __future__ import absolute_import, division, print_function - -import calendar -import collections -import copy -import datetime -import email.utils -import numbers -import re -import time -import unicodedata -import warnings - -from tornado.escape import native_str, parse_qs_bytes, utf8 -from tornado.log import gen_log -from tornado.util import ObjectDict, PY3, unicode_type - -if PY3: - import http.cookies as Cookie - from http.client import responses - from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl -else: - import Cookie - from httplib import responses - from urllib import urlencode - from urlparse import urlparse, urlunparse, parse_qsl - - -# responses is unused in this file, but we re-export it to other files. -# Reference it so pyflakes doesn't complain. -responses - -try: - from ssl import SSLError -except ImportError: - # ssl is unavailable on app engine. - class _SSLError(Exception): - pass - # Hack around a mypy limitation. We can't simply put "type: ignore" - # on the class definition itself; must go through an assignment. - SSLError = _SSLError # type: ignore - -try: - import typing # noqa: F401 -except ImportError: - pass - - -# RFC 7230 section 3.5: a recipient MAY recognize a single LF as a line -# terminator and ignore any preceding CR. -_CRLF_RE = re.compile(r'\r?\n') - - -class _NormalizedHeaderCache(dict): - """Dynamic cached mapping of header names to Http-Header-Case. - - Implemented as a dict subclass so that cache hits are as fast as a - normal dict lookup, without the overhead of a python function - call. - - >>> normalized_headers = _NormalizedHeaderCache(10) - >>> normalized_headers["coNtent-TYPE"] - 'Content-Type' - """ - def __init__(self, size): - super(_NormalizedHeaderCache, self).__init__() - self.size = size - self.queue = collections.deque() - - def __missing__(self, key): - normalized = "-".join([w.capitalize() for w in key.split("-")]) - self[key] = normalized - self.queue.append(key) - if len(self.queue) > self.size: - # Limit the size of the cache. LRU would be better, but this - # simpler approach should be fine. In Python 2.7+ we could - # use OrderedDict (or in 3.2+, @functools.lru_cache). - old_key = self.queue.popleft() - del self[old_key] - return normalized - - -_normalized_headers = _NormalizedHeaderCache(1000) - - -class HTTPHeaders(collections.MutableMapping): - """A dictionary that maintains ``Http-Header-Case`` for all keys. - - Supports multiple values per key via a pair of new methods, - `add()` and `get_list()`. The regular dictionary interface - returns a single value per key, with multiple values joined by a - comma. - - >>> h = HTTPHeaders({"content-type": "text/html"}) - >>> list(h.keys()) - ['Content-Type'] - >>> h["Content-Type"] - 'text/html' - - >>> h.add("Set-Cookie", "A=B") - >>> h.add("Set-Cookie", "C=D") - >>> h["set-cookie"] - 'A=B,C=D' - >>> h.get_list("set-cookie") - ['A=B', 'C=D'] - - >>> for (k,v) in sorted(h.get_all()): - ... print('%s: %s' % (k,v)) - ... - Content-Type: text/html - Set-Cookie: A=B - Set-Cookie: C=D - """ - def __init__(self, *args, **kwargs): - self._dict = {} # type: typing.Dict[str, str] - self._as_list = {} # type: typing.Dict[str, typing.List[str]] - self._last_key = None - if (len(args) == 1 and len(kwargs) == 0 and - isinstance(args[0], HTTPHeaders)): - # Copy constructor - for k, v in args[0].get_all(): - self.add(k, v) - else: - # Dict-style initialization - self.update(*args, **kwargs) - - # new public methods - - def add(self, name, value): - # type: (str, str) -> None - """Adds a new value for the given key.""" - norm_name = _normalized_headers[name] - self._last_key = norm_name - if norm_name in self: - self._dict[norm_name] = (native_str(self[norm_name]) + ',' + - native_str(value)) - self._as_list[norm_name].append(value) - else: - self[norm_name] = value - - def get_list(self, name): - """Returns all values for the given header as a list.""" - norm_name = _normalized_headers[name] - return self._as_list.get(norm_name, []) - - def get_all(self): - # type: () -> typing.Iterable[typing.Tuple[str, str]] - """Returns an iterable of all (name, value) pairs. - - If a header has multiple values, multiple pairs will be - returned with the same name. - """ - for name, values in self._as_list.items(): - for value in values: - yield (name, value) - - def parse_line(self, line): - """Updates the dictionary with a single header line. - - >>> h = HTTPHeaders() - >>> h.parse_line("Content-Type: text/html") - >>> h.get('content-type') - 'text/html' - """ - if line[0].isspace(): - # continuation of a multi-line header - if self._last_key is None: - raise HTTPInputError("first header line cannot start with whitespace") - new_part = ' ' + line.lstrip() - self._as_list[self._last_key][-1] += new_part - self._dict[self._last_key] += new_part - else: - try: - name, value = line.split(":", 1) - except ValueError: - raise HTTPInputError("no colon in header line") - self.add(name, value.strip()) - - @classmethod - def parse(cls, headers): - """Returns a dictionary from HTTP header text. - - >>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n") - >>> sorted(h.items()) - [('Content-Length', '42'), ('Content-Type', 'text/html')] - - .. versionchanged:: 5.1 - - Raises `HTTPInputError` on malformed headers instead of a - mix of `KeyError`, and `ValueError`. - - """ - h = cls() - for line in _CRLF_RE.split(headers): - if line: - h.parse_line(line) - return h - - # MutableMapping abstract method implementations. - - def __setitem__(self, name, value): - norm_name = _normalized_headers[name] - self._dict[norm_name] = value - self._as_list[norm_name] = [value] - - def __getitem__(self, name): - # type: (str) -> str - return self._dict[_normalized_headers[name]] - - def __delitem__(self, name): - norm_name = _normalized_headers[name] - del self._dict[norm_name] - del self._as_list[norm_name] - - def __len__(self): - return len(self._dict) - - def __iter__(self): - return iter(self._dict) - - def copy(self): - # defined in dict but not in MutableMapping. - return HTTPHeaders(self) - - # Use our overridden copy method for the copy.copy module. - # This makes shallow copies one level deeper, but preserves - # the appearance that HTTPHeaders is a single container. - __copy__ = copy - - def __str__(self): - lines = [] - for name, value in self.get_all(): - lines.append("%s: %s\n" % (name, value)) - return "".join(lines) - - __unicode__ = __str__ - - -class HTTPServerRequest(object): - """A single HTTP request. - - All attributes are type `str` unless otherwise noted. - - .. attribute:: method - - HTTP request method, e.g. "GET" or "POST" - - .. attribute:: uri - - The requested uri. - - .. attribute:: path - - The path portion of `uri` - - .. attribute:: query - - The query portion of `uri` - - .. attribute:: version - - HTTP version specified in request, e.g. "HTTP/1.1" - - .. attribute:: headers - - `.HTTPHeaders` dictionary-like object for request headers. Acts like - a case-insensitive dictionary with additional methods for repeated - headers. - - .. attribute:: body - - Request body, if present, as a byte string. - - .. attribute:: remote_ip - - Client's IP address as a string. If ``HTTPServer.xheaders`` is set, - will pass along the real IP address provided by a load balancer - in the ``X-Real-Ip`` or ``X-Forwarded-For`` header. - - .. versionchanged:: 3.1 - The list format of ``X-Forwarded-For`` is now supported. - - .. attribute:: protocol - - The protocol used, either "http" or "https". If ``HTTPServer.xheaders`` - is set, will pass along the protocol used by a load balancer if - reported via an ``X-Scheme`` header. - - .. attribute:: host - - The requested hostname, usually taken from the ``Host`` header. - - .. attribute:: arguments - - GET/POST arguments are available in the arguments property, which - maps arguments names to lists of values (to support multiple values - for individual names). Names are of type `str`, while arguments - are byte strings. Note that this is different from - `.RequestHandler.get_argument`, which returns argument values as - unicode strings. - - .. attribute:: query_arguments - - Same format as ``arguments``, but contains only arguments extracted - from the query string. - - .. versionadded:: 3.2 - - .. attribute:: body_arguments - - Same format as ``arguments``, but contains only arguments extracted - from the request body. - - .. versionadded:: 3.2 - - .. attribute:: files - - File uploads are available in the files property, which maps file - names to lists of `.HTTPFile`. - - .. attribute:: connection - - An HTTP request is attached to a single HTTP connection, which can - be accessed through the "connection" attribute. Since connections - are typically kept open in HTTP/1.1, multiple requests can be handled - sequentially on a single connection. - - .. versionchanged:: 4.0 - Moved from ``tornado.httpserver.HTTPRequest``. - """ - def __init__(self, method=None, uri=None, version="HTTP/1.0", headers=None, - body=None, host=None, files=None, connection=None, - start_line=None, server_connection=None): - if start_line is not None: - method, uri, version = start_line - self.method = method - self.uri = uri - self.version = version - self.headers = headers or HTTPHeaders() - self.body = body or b"" - - # set remote IP and protocol - context = getattr(connection, 'context', None) - self.remote_ip = getattr(context, 'remote_ip', None) - self.protocol = getattr(context, 'protocol', "http") - - self.host = host or self.headers.get("Host") or "127.0.0.1" - self.host_name = split_host_and_port(self.host.lower())[0] - self.files = files or {} - self.connection = connection - self.server_connection = server_connection - self._start_time = time.time() - self._finish_time = None - - self.path, sep, self.query = uri.partition('?') - self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) - self.query_arguments = copy.deepcopy(self.arguments) - self.body_arguments = {} - - def supports_http_1_1(self): - """Returns True if this request supports HTTP/1.1 semantics. - - .. deprecated:: 4.0 - - Applications are less likely to need this information with - the introduction of `.HTTPConnection`. If you still need - it, access the ``version`` attribute directly. This method - will be removed in Tornado 6.0. - - """ - warnings.warn("supports_http_1_1() is deprecated, use request.version instead", - DeprecationWarning) - return self.version == "HTTP/1.1" - - @property - def cookies(self): - """A dictionary of Cookie.Morsel objects.""" - if not hasattr(self, "_cookies"): - self._cookies = Cookie.SimpleCookie() - if "Cookie" in self.headers: - try: - parsed = parse_cookie(self.headers["Cookie"]) - except Exception: - pass - else: - for k, v in parsed.items(): - try: - self._cookies[k] = v - except Exception: - # SimpleCookie imposes some restrictions on keys; - # parse_cookie does not. Discard any cookies - # with disallowed keys. - pass - return self._cookies - - def write(self, chunk, callback=None): - """Writes the given chunk to the response stream. - - .. deprecated:: 4.0 - Use ``request.connection`` and the `.HTTPConnection` methods - to write the response. This method will be removed in Tornado 6.0. - """ - warnings.warn("req.write deprecated, use req.connection.write and write_headers instead", - DeprecationWarning) - assert isinstance(chunk, bytes) - assert self.version.startswith("HTTP/1."), \ - "deprecated interface only supported in HTTP/1.x" - self.connection.write(chunk, callback=callback) - - def finish(self): - """Finishes this HTTP request on the open connection. - - .. deprecated:: 4.0 - Use ``request.connection`` and the `.HTTPConnection` methods - to write the response. This method will be removed in Tornado 6.0. - """ - warnings.warn("req.finish deprecated, use req.connection.finish instead", - DeprecationWarning) - self.connection.finish() - self._finish_time = time.time() - - def full_url(self): - """Reconstructs the full URL for this request.""" - return self.protocol + "://" + self.host + self.uri - - def request_time(self): - """Returns the amount of time it took for this request to execute.""" - if self._finish_time is None: - return time.time() - self._start_time - else: - return self._finish_time - self._start_time - - def get_ssl_certificate(self, binary_form=False): - """Returns the client's SSL certificate, if any. - - To use client certificates, the HTTPServer's - `ssl.SSLContext.verify_mode` field must be set, e.g.:: - - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain("foo.crt", "foo.key") - ssl_ctx.load_verify_locations("cacerts.pem") - ssl_ctx.verify_mode = ssl.CERT_REQUIRED - server = HTTPServer(app, ssl_options=ssl_ctx) - - By default, the return value is a dictionary (or None, if no - client certificate is present). If ``binary_form`` is true, a - DER-encoded form of the certificate is returned instead. See - SSLSocket.getpeercert() in the standard library for more - details. - http://docs.python.org/library/ssl.html#sslsocket-objects - """ - try: - return self.connection.stream.socket.getpeercert( - binary_form=binary_form) - except SSLError: - return None - - def _parse_body(self): - parse_body_arguments( - self.headers.get("Content-Type", ""), self.body, - self.body_arguments, self.files, - self.headers) - - for k, v in self.body_arguments.items(): - self.arguments.setdefault(k, []).extend(v) - - def __repr__(self): - attrs = ("protocol", "host", "method", "uri", "version", "remote_ip") - args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs]) - return "%s(%s)" % (self.__class__.__name__, args) - - -class HTTPInputError(Exception): - """Exception class for malformed HTTP requests or responses - from remote sources. - - .. versionadded:: 4.0 - """ - pass - - -class HTTPOutputError(Exception): - """Exception class for errors in HTTP output. - - .. versionadded:: 4.0 - """ - pass - - -class HTTPServerConnectionDelegate(object): - """Implement this interface to handle requests from `.HTTPServer`. - - .. versionadded:: 4.0 - """ - def start_request(self, server_conn, request_conn): - """This method is called by the server when a new request has started. - - :arg server_conn: is an opaque object representing the long-lived - (e.g. tcp-level) connection. - :arg request_conn: is a `.HTTPConnection` object for a single - request/response exchange. - - This method should return a `.HTTPMessageDelegate`. - """ - raise NotImplementedError() - - def on_close(self, server_conn): - """This method is called when a connection has been closed. - - :arg server_conn: is a server connection that has previously been - passed to ``start_request``. - """ - pass - - -class HTTPMessageDelegate(object): - """Implement this interface to handle an HTTP request or response. - - .. versionadded:: 4.0 - """ - def headers_received(self, start_line, headers): - """Called when the HTTP headers have been received and parsed. - - :arg start_line: a `.RequestStartLine` or `.ResponseStartLine` - depending on whether this is a client or server message. - :arg headers: a `.HTTPHeaders` instance. - - Some `.HTTPConnection` methods can only be called during - ``headers_received``. - - May return a `.Future`; if it does the body will not be read - until it is done. - """ - pass - - def data_received(self, chunk): - """Called when a chunk of data has been received. - - May return a `.Future` for flow control. - """ - pass - - def finish(self): - """Called after the last chunk of data has been received.""" - pass - - def on_connection_close(self): - """Called if the connection is closed without finishing the request. - - If ``headers_received`` is called, either ``finish`` or - ``on_connection_close`` will be called, but not both. - """ - pass - - -class HTTPConnection(object): - """Applications use this interface to write their responses. - - .. versionadded:: 4.0 - """ - def write_headers(self, start_line, headers, chunk=None, callback=None): - """Write an HTTP header block. - - :arg start_line: a `.RequestStartLine` or `.ResponseStartLine`. - :arg headers: a `.HTTPHeaders` instance. - :arg chunk: the first (optional) chunk of data. This is an optimization - so that small responses can be written in the same call as their - headers. - :arg callback: a callback to be run when the write is complete. - - The ``version`` field of ``start_line`` is ignored. - - Returns a `.Future` if no callback is given. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. - """ - raise NotImplementedError() - - def write(self, chunk, callback=None): - """Writes a chunk of body data. - - The callback will be run when the write is complete. If no callback - is given, returns a Future. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. - """ - raise NotImplementedError() - - def finish(self): - """Indicates that the last body data has been written. - """ - raise NotImplementedError() - - -def url_concat(url, args): - """Concatenate url and arguments regardless of whether - url has existing query parameters. - - ``args`` may be either a dictionary or a list of key-value pairs - (the latter allows for multiple values with the same key. - - >>> url_concat("http://example.com/foo", dict(c="d")) - 'http://example.com/foo?c=d' - >>> url_concat("http://example.com/foo?a=b", dict(c="d")) - 'http://example.com/foo?a=b&c=d' - >>> url_concat("http://example.com/foo?a=b", [("c", "d"), ("c", "d2")]) - 'http://example.com/foo?a=b&c=d&c=d2' - """ - if args is None: - return url - parsed_url = urlparse(url) - if isinstance(args, dict): - parsed_query = parse_qsl(parsed_url.query, keep_blank_values=True) - parsed_query.extend(args.items()) - elif isinstance(args, list) or isinstance(args, tuple): - parsed_query = parse_qsl(parsed_url.query, keep_blank_values=True) - parsed_query.extend(args) - else: - err = "'args' parameter should be dict, list or tuple. Not {0}".format( - type(args)) - raise TypeError(err) - final_query = urlencode(parsed_query) - url = urlunparse(( - parsed_url[0], - parsed_url[1], - parsed_url[2], - parsed_url[3], - final_query, - parsed_url[5])) - return url - - -class HTTPFile(ObjectDict): - """Represents a file uploaded via a form. - - For backwards compatibility, its instance attributes are also - accessible as dictionary keys. - - * ``filename`` - * ``body`` - * ``content_type`` - """ - pass - - -def _parse_request_range(range_header): - """Parses a Range header. - - Returns either ``None`` or tuple ``(start, end)``. - Note that while the HTTP headers use inclusive byte positions, - this method returns indexes suitable for use in slices. - - >>> start, end = _parse_request_range("bytes=1-2") - >>> start, end - (1, 3) - >>> [0, 1, 2, 3, 4][start:end] - [1, 2] - >>> _parse_request_range("bytes=6-") - (6, None) - >>> _parse_request_range("bytes=-6") - (-6, None) - >>> _parse_request_range("bytes=-0") - (None, 0) - >>> _parse_request_range("bytes=") - (None, None) - >>> _parse_request_range("foo=42") - >>> _parse_request_range("bytes=1-2,6-10") - - Note: only supports one range (ex, ``bytes=1-2,6-10`` is not allowed). - - See [0] for the details of the range header. - - [0]: http://greenbytes.de/tech/webdav/draft-ietf-httpbis-p5-range-latest.html#byte.ranges - """ - unit, _, value = range_header.partition("=") - unit, value = unit.strip(), value.strip() - if unit != "bytes": - return None - start_b, _, end_b = value.partition("-") - try: - start = _int_or_none(start_b) - end = _int_or_none(end_b) - except ValueError: - return None - if end is not None: - if start is None: - if end != 0: - start = -end - end = None - else: - end += 1 - return (start, end) - - -def _get_content_range(start, end, total): - """Returns a suitable Content-Range header: - - >>> print(_get_content_range(None, 1, 4)) - bytes 0-0/4 - >>> print(_get_content_range(1, 3, 4)) - bytes 1-2/4 - >>> print(_get_content_range(None, None, 4)) - bytes 0-3/4 - """ - start = start or 0 - end = (end or total) - 1 - return "bytes %s-%s/%s" % (start, end, total) - - -def _int_or_none(val): - val = val.strip() - if val == "": - return None - return int(val) - - -def parse_body_arguments(content_type, body, arguments, files, headers=None): - """Parses a form request body. - - Supports ``application/x-www-form-urlencoded`` and - ``multipart/form-data``. The ``content_type`` parameter should be - a string and ``body`` should be a byte string. The ``arguments`` - and ``files`` parameters are dictionaries that will be updated - with the parsed contents. - """ - if headers and 'Content-Encoding' in headers: - gen_log.warning("Unsupported Content-Encoding: %s", - headers['Content-Encoding']) - return - if content_type.startswith("application/x-www-form-urlencoded"): - try: - uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) - except Exception as e: - gen_log.warning('Invalid x-www-form-urlencoded body: %s', e) - uri_arguments = {} - for name, values in uri_arguments.items(): - if values: - arguments.setdefault(name, []).extend(values) - elif content_type.startswith("multipart/form-data"): - try: - fields = content_type.split(";") - for field in fields: - k, sep, v = field.strip().partition("=") - if k == "boundary" and v: - parse_multipart_form_data(utf8(v), body, arguments, files) - break - else: - raise ValueError("multipart boundary not found") - except Exception as e: - gen_log.warning("Invalid multipart/form-data: %s", e) - - -def parse_multipart_form_data(boundary, data, arguments, files): - """Parses a ``multipart/form-data`` body. - - The ``boundary`` and ``data`` parameters are both byte strings. - The dictionaries given in the arguments and files parameters - will be updated with the contents of the body. - - .. versionchanged:: 5.1 - - Now recognizes non-ASCII filenames in RFC 2231/5987 - (``filename*=``) format. - """ - # The standard allows for the boundary to be quoted in the header, - # although it's rare (it happens at least for google app engine - # xmpp). I think we're also supposed to handle backslash-escapes - # here but I'll save that until we see a client that uses them - # in the wild. - if boundary.startswith(b'"') and boundary.endswith(b'"'): - boundary = boundary[1:-1] - final_boundary_index = data.rfind(b"--" + boundary + b"--") - if final_boundary_index == -1: - gen_log.warning("Invalid multipart/form-data: no final boundary") - return - parts = data[:final_boundary_index].split(b"--" + boundary + b"\r\n") - for part in parts: - if not part: - continue - eoh = part.find(b"\r\n\r\n") - if eoh == -1: - gen_log.warning("multipart/form-data missing headers") - continue - headers = HTTPHeaders.parse(part[:eoh].decode("utf-8")) - disp_header = headers.get("Content-Disposition", "") - disposition, disp_params = _parse_header(disp_header) - if disposition != "form-data" or not part.endswith(b"\r\n"): - gen_log.warning("Invalid multipart/form-data") - continue - value = part[eoh + 4:-2] - if not disp_params.get("name"): - gen_log.warning("multipart/form-data value missing name") - continue - name = disp_params["name"] - if disp_params.get("filename"): - ctype = headers.get("Content-Type", "application/unknown") - files.setdefault(name, []).append(HTTPFile( # type: ignore - filename=disp_params["filename"], body=value, - content_type=ctype)) - else: - arguments.setdefault(name, []).append(value) - - -def format_timestamp(ts): - """Formats a timestamp in the format used by HTTP. - - The argument may be a numeric timestamp as returned by `time.time`, - a time tuple as returned by `time.gmtime`, or a `datetime.datetime` - object. - - >>> format_timestamp(1359312200) - 'Sun, 27 Jan 2013 18:43:20 GMT' - """ - if isinstance(ts, numbers.Real): - pass - elif isinstance(ts, (tuple, time.struct_time)): - ts = calendar.timegm(ts) - elif isinstance(ts, datetime.datetime): - ts = calendar.timegm(ts.utctimetuple()) - else: - raise TypeError("unknown timestamp type: %r" % ts) - return email.utils.formatdate(ts, usegmt=True) - - -RequestStartLine = collections.namedtuple( - 'RequestStartLine', ['method', 'path', 'version']) - - -def parse_request_start_line(line): - """Returns a (method, path, version) tuple for an HTTP 1.x request line. - - The response is a `collections.namedtuple`. - - >>> parse_request_start_line("GET /foo HTTP/1.1") - RequestStartLine(method='GET', path='/foo', version='HTTP/1.1') - """ - try: - method, path, version = line.split(" ") - except ValueError: - # https://tools.ietf.org/html/rfc7230#section-3.1.1 - # invalid request-line SHOULD respond with a 400 (Bad Request) - raise HTTPInputError("Malformed HTTP request line") - if not re.match(r"^HTTP/1\.[0-9]$", version): - raise HTTPInputError( - "Malformed HTTP version in HTTP Request-Line: %r" % version) - return RequestStartLine(method, path, version) - - -ResponseStartLine = collections.namedtuple( - 'ResponseStartLine', ['version', 'code', 'reason']) - - -def parse_response_start_line(line): - """Returns a (version, code, reason) tuple for an HTTP 1.x response line. - - The response is a `collections.namedtuple`. - - >>> parse_response_start_line("HTTP/1.1 200 OK") - ResponseStartLine(version='HTTP/1.1', code=200, reason='OK') - """ - line = native_str(line) - match = re.match("(HTTP/1.[0-9]) ([0-9]+) ([^\r]*)", line) - if not match: - raise HTTPInputError("Error parsing response start line") - return ResponseStartLine(match.group(1), int(match.group(2)), - match.group(3)) - -# _parseparam and _parse_header are copied and modified from python2.7's cgi.py -# The original 2.7 version of this code did not correctly support some -# combinations of semicolons and double quotes. -# It has also been modified to support valueless parameters as seen in -# websocket extension negotiations, and to support non-ascii values in -# RFC 2231/5987 format. - - -def _parseparam(s): - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - yield f.strip() - s = s[end:] - - -def _parse_header(line): - r"""Parse a Content-type like header. - - Return the main content-type and a dictionary of options. - - >>> d = "form-data; foo=\"b\\\\a\\\"r\"; file*=utf-8''T%C3%A4st" - >>> ct, d = _parse_header(d) - >>> ct - 'form-data' - >>> d['file'] == r'T\u00e4st'.encode('ascii').decode('unicode_escape') - True - >>> d['foo'] - 'b\\a"r' - """ - parts = _parseparam(';' + line) - key = next(parts) - # decode_params treats first argument special, but we already stripped key - params = [('Dummy', 'value')] - for p in parts: - i = p.find('=') - if i >= 0: - name = p[:i].strip().lower() - value = p[i + 1:].strip() - params.append((name, native_str(value))) - params = email.utils.decode_params(params) - params.pop(0) # get rid of the dummy again - pdict = {} - for name, value in params: - value = email.utils.collapse_rfc2231_value(value) - if len(value) >= 2 and value[0] == '"' and value[-1] == '"': - value = value[1:-1] - pdict[name] = value - return key, pdict - - -def _encode_header(key, pdict): - """Inverse of _parse_header. - - >>> _encode_header('permessage-deflate', - ... {'client_max_window_bits': 15, 'client_no_context_takeover': None}) - 'permessage-deflate; client_max_window_bits=15; client_no_context_takeover' - """ - if not pdict: - return key - out = [key] - # Sort the parameters just to make it easy to test. - for k, v in sorted(pdict.items()): - if v is None: - out.append(k) - else: - # TODO: quote if necessary. - out.append('%s=%s' % (k, v)) - return '; '.join(out) - - -def encode_username_password(username, password): - """Encodes a username/password pair in the format used by HTTP auth. - - The return value is a byte string in the form ``username:password``. - - .. versionadded:: 5.1 - """ - if isinstance(username, unicode_type): - username = unicodedata.normalize('NFC', username) - if isinstance(password, unicode_type): - password = unicodedata.normalize('NFC', password) - return utf8(username) + b":" + utf8(password) - - -def doctests(): - import doctest - return doctest.DocTestSuite() - - -def split_host_and_port(netloc): - """Returns ``(host, port)`` tuple from ``netloc``. - - Returned ``port`` will be ``None`` if not present. - - .. versionadded:: 4.1 - """ - match = re.match(r'^(.+):(\d+)$', netloc) - if match: - host = match.group(1) - port = int(match.group(2)) - else: - host = netloc - port = None - return (host, port) - - -def qs_to_qsl(qs): - """Generator converting a result of ``parse_qs`` back to name-value pairs. - - .. versionadded:: 5.0 - """ - for k, vs in qs.items(): - for v in vs: - yield (k, v) - - -_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") -_QuotePatt = re.compile(r"[\\].") -_nulljoin = ''.join - - -def _unquote_cookie(str): - """Handle double quotes and escaping in cookie values. - - This method is copied verbatim from the Python 3.5 standard - library (http.cookies._unquote) so we don't have to depend on - non-public interfaces. - """ - # If there aren't any doublequotes, - # then there can't be any special characters. See RFC 2109. - if str is None or len(str) < 2: - return str - if str[0] != '"' or str[-1] != '"': - return str - - # We have to assume that we must decode this string. - # Down to work. - - # Remove the "s - str = str[1:-1] - - # Check for special sequences. Examples: - # \012 --> \n - # \" --> " - # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k + 1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j + 1:j + 4], 8))) - i = j + 4 - return _nulljoin(res) - - -def parse_cookie(cookie): - """Parse a ``Cookie`` HTTP header into a dict of name/value pairs. - - This function attempts to mimic browser cookie parsing behavior; - it specifically does not follow any of the cookie-related RFCs - (because browsers don't either). - - The algorithm used is identical to that used by Django version 1.9.10. - - .. versionadded:: 4.4.2 - """ - cookiedict = {} - for chunk in cookie.split(str(';')): - if str('=') in chunk: - key, val = chunk.split(str('='), 1) - else: - # Assume an empty name per - # https://bugzilla.mozilla.org/show_bug.cgi?id=169091 - key, val = str(''), chunk - key, val = key.strip(), val.strip() - if key or val: - # unquote using Python's algorithm. - cookiedict[key] = _unquote_cookie(val) - return cookiedict diff --git a/lib/tornado/ioloop.py b/lib/tornado/ioloop.py deleted file mode 100755 index 889153af..00000000 --- a/lib/tornado/ioloop.py +++ /dev/null @@ -1,1267 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""An I/O event loop for non-blocking sockets. - -On Python 3, `.IOLoop` is a wrapper around the `asyncio` event loop. - -Typical applications will use a single `IOLoop` object, accessed via -`IOLoop.current` class method. The `IOLoop.start` method (or -equivalently, `asyncio.AbstractEventLoop.run_forever`) should usually -be called at the end of the ``main()`` function. Atypical applications -may use more than one `IOLoop`, such as one `IOLoop` per thread, or -per `unittest` case. - -In addition to I/O events, the `IOLoop` can also schedule time-based -events. `IOLoop.add_timeout` is a non-blocking alternative to -`time.sleep`. - -""" - -from __future__ import absolute_import, division, print_function - -import collections -import datetime -import errno -import functools -import heapq -import itertools -import logging -import numbers -import os -import select -import sys -import threading -import time -import traceback -import math -import random - -from tornado.concurrent import Future, is_future, chain_future, future_set_exc_info, future_add_done_callback # noqa: E501 -from tornado.log import app_log, gen_log -from tornado.platform.auto import set_close_exec, Waker -from tornado import stack_context -from tornado.util import ( - PY3, Configurable, errno_from_exception, timedelta_to_seconds, - TimeoutError, unicode_type, import_object, -) - -try: - import signal -except ImportError: - signal = None - -try: - from concurrent.futures import ThreadPoolExecutor -except ImportError: - ThreadPoolExecutor = None - -if PY3: - import _thread as thread -else: - import thread - -try: - import asyncio -except ImportError: - asyncio = None - - -_POLL_TIMEOUT = 3600.0 - - -class IOLoop(Configurable): - """A level-triggered I/O loop. - - On Python 3, `IOLoop` is a wrapper around the `asyncio` event - loop. On Python 2, it uses ``epoll`` (Linux) or ``kqueue`` (BSD - and Mac OS X) if they are available, or else we fall back on - select(). If you are implementing a system that needs to handle - thousands of simultaneous connections, you should use a system - that supports either ``epoll`` or ``kqueue``. - - Example usage for a simple TCP server: - - .. testcode:: - - import errno - import functools - import socket - - import tornado.ioloop - from tornado.iostream import IOStream - - async def handle_connection(connection, address): - stream = IOStream(connection) - message = await stream.read_until_close() - print("message from client:", message.decode().strip()) - - def connection_ready(sock, fd, events): - while True: - try: - connection, address = sock.accept() - except socket.error as e: - if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN): - raise - return - connection.setblocking(0) - io_loop = tornado.ioloop.IOLoop.current() - io_loop.spawn_callback(handle_connection, connection, address) - - if __name__ == '__main__': - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setblocking(0) - sock.bind(("", 8888)) - sock.listen(128) - - io_loop = tornado.ioloop.IOLoop.current() - callback = functools.partial(connection_ready, sock) - io_loop.add_handler(sock.fileno(), callback, io_loop.READ) - io_loop.start() - - .. testoutput:: - :hide: - - By default, a newly-constructed `IOLoop` becomes the thread's current - `IOLoop`, unless there already is a current `IOLoop`. This behavior - can be controlled with the ``make_current`` argument to the `IOLoop` - constructor: if ``make_current=True``, the new `IOLoop` will always - try to become current and it raises an error if there is already a - current instance. If ``make_current=False``, the new `IOLoop` will - not try to become current. - - In general, an `IOLoop` cannot survive a fork or be shared across - processes in any way. When multiple processes are being used, each - process should create its own `IOLoop`, which also implies that - any objects which depend on the `IOLoop` (such as - `.AsyncHTTPClient`) must also be created in the child processes. - As a guideline, anything that starts processes (including the - `tornado.process` and `multiprocessing` modules) should do so as - early as possible, ideally the first thing the application does - after loading its configuration in ``main()``. - - .. versionchanged:: 4.2 - Added the ``make_current`` keyword argument to the `IOLoop` - constructor. - - .. versionchanged:: 5.0 - - Uses the `asyncio` event loop by default. The - ``IOLoop.configure`` method cannot be used on Python 3 except - to redundantly specify the `asyncio` event loop. - - """ - # Constants from the epoll module - _EPOLLIN = 0x001 - _EPOLLPRI = 0x002 - _EPOLLOUT = 0x004 - _EPOLLERR = 0x008 - _EPOLLHUP = 0x010 - _EPOLLRDHUP = 0x2000 - _EPOLLONESHOT = (1 << 30) - _EPOLLET = (1 << 31) - - # Our events map exactly to the epoll events - NONE = 0 - READ = _EPOLLIN - WRITE = _EPOLLOUT - ERROR = _EPOLLERR | _EPOLLHUP - - # In Python 2, _current.instance points to the current IOLoop. - _current = threading.local() - - # In Python 3, _ioloop_for_asyncio maps from asyncio loops to IOLoops. - _ioloop_for_asyncio = dict() - - @classmethod - def configure(cls, impl, **kwargs): - if asyncio is not None: - from tornado.platform.asyncio import BaseAsyncIOLoop - - if isinstance(impl, (str, unicode_type)): - impl = import_object(impl) - if not issubclass(impl, BaseAsyncIOLoop): - raise RuntimeError( - "only AsyncIOLoop is allowed when asyncio is available") - super(IOLoop, cls).configure(impl, **kwargs) - - @staticmethod - def instance(): - """Deprecated alias for `IOLoop.current()`. - - .. versionchanged:: 5.0 - - Previously, this method returned a global singleton - `IOLoop`, in contrast with the per-thread `IOLoop` returned - by `current()`. In nearly all cases the two were the same - (when they differed, it was generally used from non-Tornado - threads to communicate back to the main thread's `IOLoop`). - This distinction is not present in `asyncio`, so in order - to facilitate integration with that package `instance()` - was changed to be an alias to `current()`. Applications - using the cross-thread communications aspect of - `instance()` should instead set their own global variable - to point to the `IOLoop` they want to use. - - .. deprecated:: 5.0 - """ - return IOLoop.current() - - def install(self): - """Deprecated alias for `make_current()`. - - .. versionchanged:: 5.0 - - Previously, this method would set this `IOLoop` as the - global singleton used by `IOLoop.instance()`. Now that - `instance()` is an alias for `current()`, `install()` - is an alias for `make_current()`. - - .. deprecated:: 5.0 - """ - self.make_current() - - @staticmethod - def clear_instance(): - """Deprecated alias for `clear_current()`. - - .. versionchanged:: 5.0 - - Previously, this method would clear the `IOLoop` used as - the global singleton by `IOLoop.instance()`. Now that - `instance()` is an alias for `current()`, - `clear_instance()` is an alias for `clear_current()`. - - .. deprecated:: 5.0 - - """ - IOLoop.clear_current() - - @staticmethod - def current(instance=True): - """Returns the current thread's `IOLoop`. - - If an `IOLoop` is currently running or has been marked as - current by `make_current`, returns that instance. If there is - no current `IOLoop` and ``instance`` is true, creates one. - - .. versionchanged:: 4.1 - Added ``instance`` argument to control the fallback to - `IOLoop.instance()`. - .. versionchanged:: 5.0 - On Python 3, control of the current `IOLoop` is delegated - to `asyncio`, with this and other methods as pass-through accessors. - The ``instance`` argument now controls whether an `IOLoop` - is created automatically when there is none, instead of - whether we fall back to `IOLoop.instance()` (which is now - an alias for this method). ``instance=False`` is deprecated, - since even if we do not create an `IOLoop`, this method - may initialize the asyncio loop. - """ - if asyncio is None: - current = getattr(IOLoop._current, "instance", None) - if current is None and instance: - current = IOLoop() - if IOLoop._current.instance is not current: - raise RuntimeError("new IOLoop did not become current") - else: - try: - loop = asyncio.get_event_loop() - except (RuntimeError, AssertionError): - if not instance: - return None - raise - try: - return IOLoop._ioloop_for_asyncio[loop] - except KeyError: - if instance: - from tornado.platform.asyncio import AsyncIOMainLoop - current = AsyncIOMainLoop(make_current=True) - else: - current = None - return current - - def make_current(self): - """Makes this the `IOLoop` for the current thread. - - An `IOLoop` automatically becomes current for its thread - when it is started, but it is sometimes useful to call - `make_current` explicitly before starting the `IOLoop`, - so that code run at startup time can find the right - instance. - - .. versionchanged:: 4.1 - An `IOLoop` created while there is no current `IOLoop` - will automatically become current. - - .. versionchanged:: 5.0 - This method also sets the current `asyncio` event loop. - """ - # The asyncio event loops override this method. - assert asyncio is None - old = getattr(IOLoop._current, "instance", None) - if old is not None: - old.clear_current() - IOLoop._current.instance = self - - @staticmethod - def clear_current(): - """Clears the `IOLoop` for the current thread. - - Intended primarily for use by test frameworks in between tests. - - .. versionchanged:: 5.0 - This method also clears the current `asyncio` event loop. - """ - old = IOLoop.current(instance=False) - if old is not None: - old._clear_current_hook() - if asyncio is None: - IOLoop._current.instance = None - - def _clear_current_hook(self): - """Instance method called when an IOLoop ceases to be current. - - May be overridden by subclasses as a counterpart to make_current. - """ - pass - - @classmethod - def configurable_base(cls): - return IOLoop - - @classmethod - def configurable_default(cls): - if asyncio is not None: - from tornado.platform.asyncio import AsyncIOLoop - return AsyncIOLoop - return PollIOLoop - - def initialize(self, make_current=None): - if make_current is None: - if IOLoop.current(instance=False) is None: - self.make_current() - elif make_current: - current = IOLoop.current(instance=False) - # AsyncIO loops can already be current by this point. - if current is not None and current is not self: - raise RuntimeError("current IOLoop already exists") - self.make_current() - - def close(self, all_fds=False): - """Closes the `IOLoop`, freeing any resources used. - - If ``all_fds`` is true, all file descriptors registered on the - IOLoop will be closed (not just the ones created by the - `IOLoop` itself). - - Many applications will only use a single `IOLoop` that runs for the - entire lifetime of the process. In that case closing the `IOLoop` - is not necessary since everything will be cleaned up when the - process exits. `IOLoop.close` is provided mainly for scenarios - such as unit tests, which create and destroy a large number of - ``IOLoops``. - - An `IOLoop` must be completely stopped before it can be closed. This - means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must - be allowed to return before attempting to call `IOLoop.close()`. - Therefore the call to `close` will usually appear just after - the call to `start` rather than near the call to `stop`. - - .. versionchanged:: 3.1 - If the `IOLoop` implementation supports non-integer objects - for "file descriptors", those objects will have their - ``close`` method when ``all_fds`` is true. - """ - raise NotImplementedError() - - def add_handler(self, fd, handler, events): - """Registers the given handler to receive the given events for ``fd``. - - The ``fd`` argument may either be an integer file descriptor or - a file-like object with a ``fileno()`` method (and optionally a - ``close()`` method, which may be called when the `IOLoop` is shut - down). - - The ``events`` argument is a bitwise or of the constants - ``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``. - - When an event occurs, ``handler(fd, events)`` will be run. - - .. versionchanged:: 4.0 - Added the ability to pass file-like objects in addition to - raw file descriptors. - """ - raise NotImplementedError() - - def update_handler(self, fd, events): - """Changes the events we listen for ``fd``. - - .. versionchanged:: 4.0 - Added the ability to pass file-like objects in addition to - raw file descriptors. - """ - raise NotImplementedError() - - def remove_handler(self, fd): - """Stop listening for events on ``fd``. - - .. versionchanged:: 4.0 - Added the ability to pass file-like objects in addition to - raw file descriptors. - """ - raise NotImplementedError() - - def set_blocking_signal_threshold(self, seconds, action): - """Sends a signal if the `IOLoop` is blocked for more than - ``s`` seconds. - - Pass ``seconds=None`` to disable. Requires Python 2.6 on a unixy - platform. - - The action parameter is a Python signal handler. Read the - documentation for the `signal` module for more information. - If ``action`` is None, the process will be killed if it is - blocked for too long. - - .. deprecated:: 5.0 - - Not implemented on the `asyncio` event loop. Use the environment - variable ``PYTHONASYNCIODEBUG=1`` instead. This method will be - removed in Tornado 6.0. - """ - raise NotImplementedError() - - def set_blocking_log_threshold(self, seconds): - """Logs a stack trace if the `IOLoop` is blocked for more than - ``s`` seconds. - - Equivalent to ``set_blocking_signal_threshold(seconds, - self.log_stack)`` - - .. deprecated:: 5.0 - - Not implemented on the `asyncio` event loop. Use the environment - variable ``PYTHONASYNCIODEBUG=1`` instead. This method will be - removed in Tornado 6.0. - """ - self.set_blocking_signal_threshold(seconds, self.log_stack) - - def log_stack(self, signal, frame): - """Signal handler to log the stack trace of the current thread. - - For use with `set_blocking_signal_threshold`. - - .. deprecated:: 5.1 - - This method will be removed in Tornado 6.0. - """ - gen_log.warning('IOLoop blocked for %f seconds in\n%s', - self._blocking_signal_threshold, - ''.join(traceback.format_stack(frame))) - - def start(self): - """Starts the I/O loop. - - The loop will run until one of the callbacks calls `stop()`, which - will make the loop stop after the current event iteration completes. - """ - raise NotImplementedError() - - def _setup_logging(self): - """The IOLoop catches and logs exceptions, so it's - important that log output be visible. However, python's - default behavior for non-root loggers (prior to python - 3.2) is to print an unhelpful "no handlers could be - found" message rather than the actual log entry, so we - must explicitly configure logging if we've made it this - far without anything. - - This method should be called from start() in subclasses. - """ - if not any([logging.getLogger().handlers, - logging.getLogger('tornado').handlers, - logging.getLogger('tornado.application').handlers]): - logging.basicConfig() - - def stop(self): - """Stop the I/O loop. - - If the event loop is not currently running, the next call to `start()` - will return immediately. - - Note that even after `stop` has been called, the `IOLoop` is not - completely stopped until `IOLoop.start` has also returned. - Some work that was scheduled before the call to `stop` may still - be run before the `IOLoop` shuts down. - """ - raise NotImplementedError() - - def run_sync(self, func, timeout=None): - """Starts the `IOLoop`, runs the given function, and stops the loop. - - The function must return either an awaitable object or - ``None``. If the function returns an awaitable object, the - `IOLoop` will run until the awaitable is resolved (and - `run_sync()` will return the awaitable's result). If it raises - an exception, the `IOLoop` will stop and the exception will be - re-raised to the caller. - - The keyword-only argument ``timeout`` may be used to set - a maximum duration for the function. If the timeout expires, - a `tornado.util.TimeoutError` is raised. - - This method is useful to allow asynchronous calls in a - ``main()`` function:: - - async def main(): - # do stuff... - - if __name__ == '__main__': - IOLoop.current().run_sync(main) - - .. versionchanged:: 4.3 - Returning a non-``None``, non-awaitable value is now an error. - - .. versionchanged:: 5.0 - If a timeout occurs, the ``func`` coroutine will be cancelled. - - """ - future_cell = [None] - - def run(): - try: - result = func() - if result is not None: - from tornado.gen import convert_yielded - result = convert_yielded(result) - except Exception: - future_cell[0] = Future() - future_set_exc_info(future_cell[0], sys.exc_info()) - else: - if is_future(result): - future_cell[0] = result - else: - future_cell[0] = Future() - future_cell[0].set_result(result) - self.add_future(future_cell[0], lambda future: self.stop()) - self.add_callback(run) - if timeout is not None: - def timeout_callback(): - # If we can cancel the future, do so and wait on it. If not, - # Just stop the loop and return with the task still pending. - # (If we neither cancel nor wait for the task, a warning - # will be logged). - if not future_cell[0].cancel(): - self.stop() - timeout_handle = self.add_timeout(self.time() + timeout, timeout_callback) - self.start() - if timeout is not None: - self.remove_timeout(timeout_handle) - if future_cell[0].cancelled() or not future_cell[0].done(): - raise TimeoutError('Operation timed out after %s seconds' % timeout) - return future_cell[0].result() - - def time(self): - """Returns the current time according to the `IOLoop`'s clock. - - The return value is a floating-point number relative to an - unspecified time in the past. - - By default, the `IOLoop`'s time function is `time.time`. However, - it may be configured to use e.g. `time.monotonic` instead. - Calls to `add_timeout` that pass a number instead of a - `datetime.timedelta` should use this function to compute the - appropriate time, so they can work no matter what time function - is chosen. - """ - return time.time() - - def add_timeout(self, deadline, callback, *args, **kwargs): - """Runs the ``callback`` at the time ``deadline`` from the I/O loop. - - Returns an opaque handle that may be passed to - `remove_timeout` to cancel. - - ``deadline`` may be a number denoting a time (on the same - scale as `IOLoop.time`, normally `time.time`), or a - `datetime.timedelta` object for a deadline relative to the - current time. Since Tornado 4.0, `call_later` is a more - convenient alternative for the relative case since it does not - require a timedelta object. - - Note that it is not safe to call `add_timeout` from other threads. - Instead, you must use `add_callback` to transfer control to the - `IOLoop`'s thread, and then call `add_timeout` from there. - - Subclasses of IOLoop must implement either `add_timeout` or - `call_at`; the default implementations of each will call - the other. `call_at` is usually easier to implement, but - subclasses that wish to maintain compatibility with Tornado - versions prior to 4.0 must use `add_timeout` instead. - - .. versionchanged:: 4.0 - Now passes through ``*args`` and ``**kwargs`` to the callback. - """ - if isinstance(deadline, numbers.Real): - return self.call_at(deadline, callback, *args, **kwargs) - elif isinstance(deadline, datetime.timedelta): - return self.call_at(self.time() + timedelta_to_seconds(deadline), - callback, *args, **kwargs) - else: - raise TypeError("Unsupported deadline %r" % deadline) - - def call_later(self, delay, callback, *args, **kwargs): - """Runs the ``callback`` after ``delay`` seconds have passed. - - Returns an opaque handle that may be passed to `remove_timeout` - to cancel. Note that unlike the `asyncio` method of the same - name, the returned object does not have a ``cancel()`` method. - - See `add_timeout` for comments on thread-safety and subclassing. - - .. versionadded:: 4.0 - """ - return self.call_at(self.time() + delay, callback, *args, **kwargs) - - def call_at(self, when, callback, *args, **kwargs): - """Runs the ``callback`` at the absolute time designated by ``when``. - - ``when`` must be a number using the same reference point as - `IOLoop.time`. - - Returns an opaque handle that may be passed to `remove_timeout` - to cancel. Note that unlike the `asyncio` method of the same - name, the returned object does not have a ``cancel()`` method. - - See `add_timeout` for comments on thread-safety and subclassing. - - .. versionadded:: 4.0 - """ - return self.add_timeout(when, callback, *args, **kwargs) - - def remove_timeout(self, timeout): - """Cancels a pending timeout. - - The argument is a handle as returned by `add_timeout`. It is - safe to call `remove_timeout` even if the callback has already - been run. - """ - raise NotImplementedError() - - def add_callback(self, callback, *args, **kwargs): - """Calls the given callback on the next I/O loop iteration. - - It is safe to call this method from any thread at any time, - except from a signal handler. Note that this is the **only** - method in `IOLoop` that makes this thread-safety guarantee; all - other interaction with the `IOLoop` must be done from that - `IOLoop`'s thread. `add_callback()` may be used to transfer - control from other threads to the `IOLoop`'s thread. - - To add a callback from a signal handler, see - `add_callback_from_signal`. - """ - raise NotImplementedError() - - def add_callback_from_signal(self, callback, *args, **kwargs): - """Calls the given callback on the next I/O loop iteration. - - Safe for use from a Python signal handler; should not be used - otherwise. - - Callbacks added with this method will be run without any - `.stack_context`, to avoid picking up the context of the function - that was interrupted by the signal. - """ - raise NotImplementedError() - - def spawn_callback(self, callback, *args, **kwargs): - """Calls the given callback on the next IOLoop iteration. - - Unlike all other callback-related methods on IOLoop, - ``spawn_callback`` does not associate the callback with its caller's - ``stack_context``, so it is suitable for fire-and-forget callbacks - that should not interfere with the caller. - - .. versionadded:: 4.0 - """ - with stack_context.NullContext(): - self.add_callback(callback, *args, **kwargs) - - def add_future(self, future, callback): - """Schedules a callback on the ``IOLoop`` when the given - `.Future` is finished. - - The callback is invoked with one argument, the - `.Future`. - - This method only accepts `.Future` objects and not other - awaitables (unlike most of Tornado where the two are - interchangeable). - """ - assert is_future(future) - callback = stack_context.wrap(callback) - future_add_done_callback( - future, lambda future: self.add_callback(callback, future)) - - def run_in_executor(self, executor, func, *args): - """Runs a function in a ``concurrent.futures.Executor``. If - ``executor`` is ``None``, the IO loop's default executor will be used. - - Use `functools.partial` to pass keyword arguments to ``func``. - - .. versionadded:: 5.0 - """ - if ThreadPoolExecutor is None: - raise RuntimeError( - "concurrent.futures is required to use IOLoop.run_in_executor") - - if executor is None: - if not hasattr(self, '_executor'): - from tornado.process import cpu_count - self._executor = ThreadPoolExecutor(max_workers=(cpu_count() * 5)) - executor = self._executor - c_future = executor.submit(func, *args) - # Concurrent Futures are not usable with await. Wrap this in a - # Tornado Future instead, using self.add_future for thread-safety. - t_future = Future() - self.add_future(c_future, lambda f: chain_future(f, t_future)) - return t_future - - def set_default_executor(self, executor): - """Sets the default executor to use with :meth:`run_in_executor`. - - .. versionadded:: 5.0 - """ - self._executor = executor - - def _run_callback(self, callback): - """Runs a callback with error handling. - - For use in subclasses. - """ - try: - ret = callback() - if ret is not None: - from tornado import gen - # Functions that return Futures typically swallow all - # exceptions and store them in the Future. If a Future - # makes it out to the IOLoop, ensure its exception (if any) - # gets logged too. - try: - ret = gen.convert_yielded(ret) - except gen.BadYieldError: - # It's not unusual for add_callback to be used with - # methods returning a non-None and non-yieldable - # result, which should just be ignored. - pass - else: - self.add_future(ret, self._discard_future_result) - except Exception: - self.handle_callback_exception(callback) - - def _discard_future_result(self, future): - """Avoid unhandled-exception warnings from spawned coroutines.""" - future.result() - - def handle_callback_exception(self, callback): - """This method is called whenever a callback run by the `IOLoop` - throws an exception. - - By default simply logs the exception as an error. Subclasses - may override this method to customize reporting of exceptions. - - The exception itself is not passed explicitly, but is available - in `sys.exc_info`. - - .. versionchanged:: 5.0 - - When the `asyncio` event loop is used (which is now the - default on Python 3), some callback errors will be handled by - `asyncio` instead of this method. - - .. deprecated: 5.1 - - Support for this method will be removed in Tornado 6.0. - """ - app_log.error("Exception in callback %r", callback, exc_info=True) - - def split_fd(self, fd): - """Returns an (fd, obj) pair from an ``fd`` parameter. - - We accept both raw file descriptors and file-like objects as - input to `add_handler` and related methods. When a file-like - object is passed, we must retain the object itself so we can - close it correctly when the `IOLoop` shuts down, but the - poller interfaces favor file descriptors (they will accept - file-like objects and call ``fileno()`` for you, but they - always return the descriptor itself). - - This method is provided for use by `IOLoop` subclasses and should - not generally be used by application code. - - .. versionadded:: 4.0 - """ - try: - return fd.fileno(), fd - except AttributeError: - return fd, fd - - def close_fd(self, fd): - """Utility method to close an ``fd``. - - If ``fd`` is a file-like object, we close it directly; otherwise - we use `os.close`. - - This method is provided for use by `IOLoop` subclasses (in - implementations of ``IOLoop.close(all_fds=True)`` and should - not generally be used by application code. - - .. versionadded:: 4.0 - """ - try: - try: - fd.close() - except AttributeError: - os.close(fd) - except OSError: - pass - - -class PollIOLoop(IOLoop): - """Base class for IOLoops built around a select-like function. - - For concrete implementations, see `tornado.platform.epoll.EPollIOLoop` - (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or - `tornado.platform.select.SelectIOLoop` (all platforms). - """ - def initialize(self, impl, time_func=None, **kwargs): - super(PollIOLoop, self).initialize(**kwargs) - self._impl = impl - if hasattr(self._impl, 'fileno'): - set_close_exec(self._impl.fileno()) - self.time_func = time_func or time.time - self._handlers = {} - self._events = {} - self._callbacks = collections.deque() - self._timeouts = [] - self._cancellations = 0 - self._running = False - self._stopped = False - self._closing = False - self._thread_ident = None - self._pid = os.getpid() - self._blocking_signal_threshold = None - self._timeout_counter = itertools.count() - - # Create a pipe that we send bogus data to when we want to wake - # the I/O loop when it is idle - self._waker = Waker() - self.add_handler(self._waker.fileno(), - lambda fd, events: self._waker.consume(), - self.READ) - - @classmethod - def configurable_base(cls): - return PollIOLoop - - @classmethod - def configurable_default(cls): - if hasattr(select, "epoll"): - from tornado.platform.epoll import EPollIOLoop - return EPollIOLoop - if hasattr(select, "kqueue"): - # Python 2.6+ on BSD or Mac - from tornado.platform.kqueue import KQueueIOLoop - return KQueueIOLoop - from tornado.platform.select import SelectIOLoop - return SelectIOLoop - - def close(self, all_fds=False): - self._closing = True - self.remove_handler(self._waker.fileno()) - if all_fds: - for fd, handler in list(self._handlers.values()): - self.close_fd(fd) - self._waker.close() - self._impl.close() - self._callbacks = None - self._timeouts = None - if hasattr(self, '_executor'): - self._executor.shutdown() - - def add_handler(self, fd, handler, events): - fd, obj = self.split_fd(fd) - self._handlers[fd] = (obj, stack_context.wrap(handler)) - self._impl.register(fd, events | self.ERROR) - - def update_handler(self, fd, events): - fd, obj = self.split_fd(fd) - self._impl.modify(fd, events | self.ERROR) - - def remove_handler(self, fd): - fd, obj = self.split_fd(fd) - self._handlers.pop(fd, None) - self._events.pop(fd, None) - try: - self._impl.unregister(fd) - except Exception: - gen_log.debug("Error deleting fd from IOLoop", exc_info=True) - - def set_blocking_signal_threshold(self, seconds, action): - if not hasattr(signal, "setitimer"): - gen_log.error("set_blocking_signal_threshold requires a signal module " - "with the setitimer method") - return - self._blocking_signal_threshold = seconds - if seconds is not None: - signal.signal(signal.SIGALRM, - action if action is not None else signal.SIG_DFL) - - def start(self): - if self._running: - raise RuntimeError("IOLoop is already running") - if os.getpid() != self._pid: - raise RuntimeError("Cannot share PollIOLoops across processes") - self._setup_logging() - if self._stopped: - self._stopped = False - return - old_current = IOLoop.current(instance=False) - if old_current is not self: - self.make_current() - self._thread_ident = thread.get_ident() - self._running = True - - # signal.set_wakeup_fd closes a race condition in event loops: - # a signal may arrive at the beginning of select/poll/etc - # before it goes into its interruptible sleep, so the signal - # will be consumed without waking the select. The solution is - # for the (C, synchronous) signal handler to write to a pipe, - # which will then be seen by select. - # - # In python's signal handling semantics, this only matters on the - # main thread (fortunately, set_wakeup_fd only works on the main - # thread and will raise a ValueError otherwise). - # - # If someone has already set a wakeup fd, we don't want to - # disturb it. This is an issue for twisted, which does its - # SIGCHLD processing in response to its own wakeup fd being - # written to. As long as the wakeup fd is registered on the IOLoop, - # the loop will still wake up and everything should work. - old_wakeup_fd = None - if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': - # requires python 2.6+, unix. set_wakeup_fd exists but crashes - # the python process on windows. - try: - old_wakeup_fd = signal.set_wakeup_fd(self._waker.write_fileno()) - if old_wakeup_fd != -1: - # Already set, restore previous value. This is a little racy, - # but there's no clean get_wakeup_fd and in real use the - # IOLoop is just started once at the beginning. - signal.set_wakeup_fd(old_wakeup_fd) - old_wakeup_fd = None - except ValueError: - # Non-main thread, or the previous value of wakeup_fd - # is no longer valid. - old_wakeup_fd = None - - try: - while True: - # Prevent IO event starvation by delaying new callbacks - # to the next iteration of the event loop. - ncallbacks = len(self._callbacks) - - # Add any timeouts that have come due to the callback list. - # Do not run anything until we have determined which ones - # are ready, so timeouts that call add_timeout cannot - # schedule anything in this iteration. - due_timeouts = [] - if self._timeouts: - now = self.time() - while self._timeouts: - if self._timeouts[0].callback is None: - # The timeout was cancelled. Note that the - # cancellation check is repeated below for timeouts - # that are cancelled by another timeout or callback. - heapq.heappop(self._timeouts) - self._cancellations -= 1 - elif self._timeouts[0].deadline <= now: - due_timeouts.append(heapq.heappop(self._timeouts)) - else: - break - if (self._cancellations > 512 and - self._cancellations > (len(self._timeouts) >> 1)): - # Clean up the timeout queue when it gets large and it's - # more than half cancellations. - self._cancellations = 0 - self._timeouts = [x for x in self._timeouts - if x.callback is not None] - heapq.heapify(self._timeouts) - - for i in range(ncallbacks): - self._run_callback(self._callbacks.popleft()) - for timeout in due_timeouts: - if timeout.callback is not None: - self._run_callback(timeout.callback) - # Closures may be holding on to a lot of memory, so allow - # them to be freed before we go into our poll wait. - due_timeouts = timeout = None - - if self._callbacks: - # If any callbacks or timeouts called add_callback, - # we don't want to wait in poll() before we run them. - poll_timeout = 0.0 - elif self._timeouts: - # If there are any timeouts, schedule the first one. - # Use self.time() instead of 'now' to account for time - # spent running callbacks. - poll_timeout = self._timeouts[0].deadline - self.time() - poll_timeout = max(0, min(poll_timeout, _POLL_TIMEOUT)) - else: - # No timeouts and no callbacks, so use the default. - poll_timeout = _POLL_TIMEOUT - - if not self._running: - break - - if self._blocking_signal_threshold is not None: - # clear alarm so it doesn't fire while poll is waiting for - # events. - signal.setitimer(signal.ITIMER_REAL, 0, 0) - - try: - event_pairs = self._impl.poll(poll_timeout) - except Exception as e: - # Depending on python version and IOLoop implementation, - # different exception types may be thrown and there are - # two ways EINTR might be signaled: - # * e.errno == errno.EINTR - # * e.args is like (errno.EINTR, 'Interrupted system call') - if errno_from_exception(e) == errno.EINTR: - continue - else: - raise - - if self._blocking_signal_threshold is not None: - signal.setitimer(signal.ITIMER_REAL, - self._blocking_signal_threshold, 0) - - # Pop one fd at a time from the set of pending fds and run - # its handler. Since that handler may perform actions on - # other file descriptors, there may be reentrant calls to - # this IOLoop that modify self._events - self._events.update(event_pairs) - while self._events: - fd, events = self._events.popitem() - try: - fd_obj, handler_func = self._handlers[fd] - handler_func(fd_obj, events) - except (OSError, IOError) as e: - if errno_from_exception(e) == errno.EPIPE: - # Happens when the client closes the connection - pass - else: - self.handle_callback_exception(self._handlers.get(fd)) - except Exception: - self.handle_callback_exception(self._handlers.get(fd)) - fd_obj = handler_func = None - - finally: - # reset the stopped flag so another start/stop pair can be issued - self._stopped = False - if self._blocking_signal_threshold is not None: - signal.setitimer(signal.ITIMER_REAL, 0, 0) - if old_current is None: - IOLoop.clear_current() - elif old_current is not self: - old_current.make_current() - if old_wakeup_fd is not None: - signal.set_wakeup_fd(old_wakeup_fd) - - def stop(self): - self._running = False - self._stopped = True - self._waker.wake() - - def time(self): - return self.time_func() - - def call_at(self, deadline, callback, *args, **kwargs): - timeout = _Timeout( - deadline, - functools.partial(stack_context.wrap(callback), *args, **kwargs), - self) - heapq.heappush(self._timeouts, timeout) - return timeout - - def remove_timeout(self, timeout): - # Removing from a heap is complicated, so just leave the defunct - # timeout object in the queue (see discussion in - # http://docs.python.org/library/heapq.html). - # If this turns out to be a problem, we could add a garbage - # collection pass whenever there are too many dead timeouts. - timeout.callback = None - self._cancellations += 1 - - def add_callback(self, callback, *args, **kwargs): - if self._closing: - return - # Blindly insert into self._callbacks. This is safe even - # from signal handlers because deque.append is atomic. - self._callbacks.append(functools.partial( - stack_context.wrap(callback), *args, **kwargs)) - if thread.get_ident() != self._thread_ident: - # This will write one byte but Waker.consume() reads many - # at once, so it's ok to write even when not strictly - # necessary. - self._waker.wake() - else: - # If we're on the IOLoop's thread, we don't need to wake anyone. - pass - - def add_callback_from_signal(self, callback, *args, **kwargs): - with stack_context.NullContext(): - self.add_callback(callback, *args, **kwargs) - - -class _Timeout(object): - """An IOLoop timeout, a UNIX timestamp and a callback""" - - # Reduce memory overhead when there are lots of pending callbacks - __slots__ = ['deadline', 'callback', 'tdeadline'] - - def __init__(self, deadline, callback, io_loop): - if not isinstance(deadline, numbers.Real): - raise TypeError("Unsupported deadline %r" % deadline) - self.deadline = deadline - self.callback = callback - self.tdeadline = (deadline, next(io_loop._timeout_counter)) - - # Comparison methods to sort by deadline, with object id as a tiebreaker - # to guarantee a consistent ordering. The heapq module uses __le__ - # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons - # use __lt__). - def __lt__(self, other): - return self.tdeadline < other.tdeadline - - def __le__(self, other): - return self.tdeadline <= other.tdeadline - - -class PeriodicCallback(object): - """Schedules the given callback to be called periodically. - - The callback is called every ``callback_time`` milliseconds. - Note that the timeout is given in milliseconds, while most other - time-related functions in Tornado use seconds. - - If ``jitter`` is specified, each callback time will be randomly selected - within a window of ``jitter * callback_time`` milliseconds. - Jitter can be used to reduce alignment of events with similar periods. - A jitter of 0.1 means allowing a 10% variation in callback time. - The window is centered on ``callback_time`` so the total number of calls - within a given interval should not be significantly affected by adding - jitter. - - If the callback runs for longer than ``callback_time`` milliseconds, - subsequent invocations will be skipped to get back on schedule. - - `start` must be called after the `PeriodicCallback` is created. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - .. versionchanged:: 5.1 - The ``jitter`` argument is added. - """ - def __init__(self, callback, callback_time, jitter=0): - self.callback = callback - if callback_time <= 0: - raise ValueError("Periodic callback must have a positive callback_time") - self.callback_time = callback_time - self.jitter = jitter - self._running = False - self._timeout = None - - def start(self): - """Starts the timer.""" - # Looking up the IOLoop here allows to first instantiate the - # PeriodicCallback in another thread, then start it using - # IOLoop.add_callback(). - self.io_loop = IOLoop.current() - self._running = True - self._next_timeout = self.io_loop.time() - self._schedule_next() - - def stop(self): - """Stops the timer.""" - self._running = False - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = None - - def is_running(self): - """Return True if this `.PeriodicCallback` has been started. - - .. versionadded:: 4.1 - """ - return self._running - - def _run(self): - if not self._running: - return - try: - return self.callback() - except Exception: - self.io_loop.handle_callback_exception(self.callback) - finally: - self._schedule_next() - - def _schedule_next(self): - if self._running: - self._update_next(self.io_loop.time()) - self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run) - - def _update_next(self, current_time): - callback_time_sec = self.callback_time / 1000.0 - if self.jitter: - # apply jitter fraction - callback_time_sec *= 1 + (self.jitter * (random.random() - 0.5)) - if self._next_timeout <= current_time: - # The period should be measured from the start of one call - # to the start of the next. If one call takes too long, - # skip cycles to get back to a multiple of the original - # schedule. - self._next_timeout += (math.floor((current_time - self._next_timeout) / - callback_time_sec) + 1) * callback_time_sec - else: - # If the clock moved backwards, ensure we advance the next - # timeout instead of recomputing the same value again. - # This may result in long gaps between callbacks if the - # clock jumps backwards by a lot, but the far more common - # scenario is a small NTP adjustment that should just be - # ignored. - # - # Note that on some systems if time.time() runs slower - # than time.monotonic() (most common on windows), we - # effectively experience a small backwards time jump on - # every iteration because PeriodicCallback uses - # time.time() while asyncio schedules callbacks using - # time.monotonic(). - # https://github.com/tornadoweb/tornado/issues/2333 - self._next_timeout += callback_time_sec diff --git a/lib/tornado/iostream.py b/lib/tornado/iostream.py deleted file mode 100755 index 89e1e234..00000000 --- a/lib/tornado/iostream.py +++ /dev/null @@ -1,1757 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utility classes to write to and read from non-blocking files and sockets. - -Contents: - -* `BaseIOStream`: Generic interface for reading and writing. -* `IOStream`: Implementation of BaseIOStream using non-blocking sockets. -* `SSLIOStream`: SSL-aware version of IOStream. -* `PipeIOStream`: Pipe-based IOStream implementation. -""" - -from __future__ import absolute_import, division, print_function - -import collections -import errno -import io -import numbers -import os -import socket -import sys -import re -import warnings - -from tornado.concurrent import Future -from tornado import ioloop -from tornado.log import gen_log, app_log -from tornado.netutil import ssl_wrap_socket, _client_ssl_defaults, _server_ssl_defaults -from tornado import stack_context -from tornado.util import errno_from_exception - -try: - from tornado.platform.posix import _set_nonblocking -except ImportError: - _set_nonblocking = None - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine - ssl = None - -# These errnos indicate that a non-blocking operation must be retried -# at a later time. On most platforms they're the same value, but on -# some they differ. -_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) - -if hasattr(errno, "WSAEWOULDBLOCK"): - _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) # type: ignore - -# These errnos indicate that a connection has been abruptly terminated. -# They should be caught and handled less noisily than other errors. -_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE, - errno.ETIMEDOUT) - -if hasattr(errno, "WSAECONNRESET"): - _ERRNO_CONNRESET += (errno.WSAECONNRESET, errno.WSAECONNABORTED, errno.WSAETIMEDOUT) # type: ignore # noqa: E501 - -if sys.platform == 'darwin': - # OSX appears to have a race condition that causes send(2) to return - # EPROTOTYPE if called while a socket is being torn down: - # http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ - # Since the socket is being closed anyway, treat this as an ECONNRESET - # instead of an unexpected error. - _ERRNO_CONNRESET += (errno.EPROTOTYPE,) # type: ignore - -# More non-portable errnos: -_ERRNO_INPROGRESS = (errno.EINPROGRESS,) - -if hasattr(errno, "WSAEINPROGRESS"): - _ERRNO_INPROGRESS += (errno.WSAEINPROGRESS,) # type: ignore - -_WINDOWS = sys.platform.startswith('win') - - -class StreamClosedError(IOError): - """Exception raised by `IOStream` methods when the stream is closed. - - Note that the close callback is scheduled to run *after* other - callbacks on the stream (to allow for buffered data to be processed), - so you may see this error before you see the close callback. - - The ``real_error`` attribute contains the underlying error that caused - the stream to close (if any). - - .. versionchanged:: 4.3 - Added the ``real_error`` attribute. - """ - def __init__(self, real_error=None): - super(StreamClosedError, self).__init__('Stream is closed') - self.real_error = real_error - - -class UnsatisfiableReadError(Exception): - """Exception raised when a read cannot be satisfied. - - Raised by ``read_until`` and ``read_until_regex`` with a ``max_bytes`` - argument. - """ - pass - - -class StreamBufferFullError(Exception): - """Exception raised by `IOStream` methods when the buffer is full. - """ - - -class _StreamBuffer(object): - """ - A specialized buffer that tries to avoid copies when large pieces - of data are encountered. - """ - - def __init__(self): - # A sequence of (False, bytearray) and (True, memoryview) objects - self._buffers = collections.deque() - # Position in the first buffer - self._first_pos = 0 - self._size = 0 - - def __len__(self): - return self._size - - # Data above this size will be appended separately instead - # of extending an existing bytearray - _large_buf_threshold = 2048 - - def append(self, data): - """ - Append the given piece of data (should be a buffer-compatible object). - """ - size = len(data) - if size > self._large_buf_threshold: - if not isinstance(data, memoryview): - data = memoryview(data) - self._buffers.append((True, data)) - elif size > 0: - if self._buffers: - is_memview, b = self._buffers[-1] - new_buf = is_memview or len(b) >= self._large_buf_threshold - else: - new_buf = True - if new_buf: - self._buffers.append((False, bytearray(data))) - else: - b += data - - self._size += size - - def peek(self, size): - """ - Get a view over at most ``size`` bytes (possibly fewer) at the - current buffer position. - """ - assert size > 0 - try: - is_memview, b = self._buffers[0] - except IndexError: - return memoryview(b'') - - pos = self._first_pos - if is_memview: - return b[pos:pos + size] - else: - return memoryview(b)[pos:pos + size] - - def advance(self, size): - """ - Advance the current buffer position by ``size`` bytes. - """ - assert 0 < size <= self._size - self._size -= size - pos = self._first_pos - - buffers = self._buffers - while buffers and size > 0: - is_large, b = buffers[0] - b_remain = len(b) - size - pos - if b_remain <= 0: - buffers.popleft() - size -= len(b) - pos - pos = 0 - elif is_large: - pos += size - size = 0 - else: - # Amortized O(1) shrink for Python 2 - pos += size - if len(b) <= 2 * pos: - del b[:pos] - pos = 0 - size = 0 - - assert size == 0 - self._first_pos = pos - - -class BaseIOStream(object): - """A utility class to write to and read from a non-blocking file or socket. - - We support a non-blocking ``write()`` and a family of ``read_*()`` methods. - All of the methods take an optional ``callback`` argument and return a - `.Future` only if no callback is given. When the operation completes, - the callback will be run or the `.Future` will resolve with the data - read (or ``None`` for ``write()``). All outstanding ``Futures`` will - resolve with a `StreamClosedError` when the stream is closed; users - of the callback interface will be notified via - `.BaseIOStream.set_close_callback` instead. - - When a stream is closed due to an error, the IOStream's ``error`` - attribute contains the exception object. - - Subclasses must implement `fileno`, `close_fd`, `write_to_fd`, - `read_from_fd`, and optionally `get_fd_error`. - """ - def __init__(self, max_buffer_size=None, - read_chunk_size=None, max_write_buffer_size=None): - """`BaseIOStream` constructor. - - :arg max_buffer_size: Maximum amount of incoming data to buffer; - defaults to 100MB. - :arg read_chunk_size: Amount of data to read at one time from the - underlying transport; defaults to 64KB. - :arg max_write_buffer_size: Amount of outgoing data to buffer; - defaults to unlimited. - - .. versionchanged:: 4.0 - Add the ``max_write_buffer_size`` parameter. Changed default - ``read_chunk_size`` to 64KB. - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been - removed. - """ - self.io_loop = ioloop.IOLoop.current() - self.max_buffer_size = max_buffer_size or 104857600 - # A chunk size that is too close to max_buffer_size can cause - # spurious failures. - self.read_chunk_size = min(read_chunk_size or 65536, - self.max_buffer_size // 2) - self.max_write_buffer_size = max_write_buffer_size - self.error = None - self._read_buffer = bytearray() - self._read_buffer_pos = 0 - self._read_buffer_size = 0 - self._user_read_buffer = False - self._after_user_read_buffer = None - self._write_buffer = _StreamBuffer() - self._total_write_index = 0 - self._total_write_done_index = 0 - self._read_delimiter = None - self._read_regex = None - self._read_max_bytes = None - self._read_bytes = None - self._read_partial = False - self._read_until_close = False - self._read_callback = None - self._read_future = None - self._streaming_callback = None - self._write_callback = None - self._write_futures = collections.deque() - self._close_callback = None - self._connect_callback = None - self._connect_future = None - # _ssl_connect_future should be defined in SSLIOStream - # but it's here so we can clean it up in maybe_run_close_callback. - # TODO: refactor that so subclasses can add additional futures - # to be cancelled. - self._ssl_connect_future = None - self._connecting = False - self._state = None - self._pending_callbacks = 0 - self._closed = False - - def fileno(self): - """Returns the file descriptor for this stream.""" - raise NotImplementedError() - - def close_fd(self): - """Closes the file underlying this stream. - - ``close_fd`` is called by `BaseIOStream` and should not be called - elsewhere; other users should call `close` instead. - """ - raise NotImplementedError() - - def write_to_fd(self, data): - """Attempts to write ``data`` to the underlying file. - - Returns the number of bytes written. - """ - raise NotImplementedError() - - def read_from_fd(self, buf): - """Attempts to read from the underlying file. - - Reads up to ``len(buf)`` bytes, storing them in the buffer. - Returns the number of bytes read. Returns None if there was - nothing to read (the socket returned `~errno.EWOULDBLOCK` or - equivalent), and zero on EOF. - - .. versionchanged:: 5.0 - - Interface redesigned to take a buffer and return a number - of bytes instead of a freshly-allocated object. - """ - raise NotImplementedError() - - def get_fd_error(self): - """Returns information about any error on the underlying file. - - This method is called after the `.IOLoop` has signaled an error on the - file descriptor, and should return an Exception (such as `socket.error` - with additional information, or None if no such information is - available. - """ - return None - - def read_until_regex(self, regex, callback=None, max_bytes=None): - """Asynchronously read until we have matched the given regex. - - The result includes the data that matches the regex and anything - that came before it. If a callback is given, it will be run - with the data as an argument; if not, this method returns a - `.Future`. - - If ``max_bytes`` is not None, the connection will be closed - if more than ``max_bytes`` bytes have been read and the regex is - not satisfied. - - .. versionchanged:: 4.0 - Added the ``max_bytes`` argument. The ``callback`` argument is - now optional and a `.Future` will be returned if it is omitted. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. Use the returned `.Future` instead. - - """ - future = self._set_read_callback(callback) - self._read_regex = re.compile(regex) - self._read_max_bytes = max_bytes - try: - self._try_inline_read() - except UnsatisfiableReadError as e: - # Handle this the same way as in _handle_events. - gen_log.info("Unsatisfiable read, closing connection: %s" % e) - self.close(exc_info=e) - return future - except: - if future is not None: - # Ensure that the future doesn't log an error because its - # failure was never examined. - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_until(self, delimiter, callback=None, max_bytes=None): - """Asynchronously read until we have found the given delimiter. - - The result includes all the data read including the delimiter. - If a callback is given, it will be run with the data as an argument; - if not, this method returns a `.Future`. - - If ``max_bytes`` is not None, the connection will be closed - if more than ``max_bytes`` bytes have been read and the delimiter - is not found. - - .. versionchanged:: 4.0 - Added the ``max_bytes`` argument. The ``callback`` argument is - now optional and a `.Future` will be returned if it is omitted. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. Use the returned `.Future` instead. - """ - future = self._set_read_callback(callback) - self._read_delimiter = delimiter - self._read_max_bytes = max_bytes - try: - self._try_inline_read() - except UnsatisfiableReadError as e: - # Handle this the same way as in _handle_events. - gen_log.info("Unsatisfiable read, closing connection: %s" % e) - self.close(exc_info=e) - return future - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_bytes(self, num_bytes, callback=None, streaming_callback=None, - partial=False): - """Asynchronously read a number of bytes. - - If a ``streaming_callback`` is given, it will be called with chunks - of data as they become available, and the final result will be empty. - Otherwise, the result is all the data that was read. - If a callback is given, it will be run with the data as an argument; - if not, this method returns a `.Future`. - - If ``partial`` is true, the callback is run as soon as we have - any bytes to return (but never more than ``num_bytes``) - - .. versionchanged:: 4.0 - Added the ``partial`` argument. The callback argument is now - optional and a `.Future` will be returned if it is omitted. - - .. deprecated:: 5.1 - - The ``callback`` and ``streaming_callback`` arguments are - deprecated and will be removed in Tornado 6.0. Use the - returned `.Future` (and ``partial=True`` for - ``streaming_callback``) instead. - - """ - future = self._set_read_callback(callback) - assert isinstance(num_bytes, numbers.Integral) - self._read_bytes = num_bytes - self._read_partial = partial - if streaming_callback is not None: - warnings.warn("streaming_callback is deprecated, use partial instead", - DeprecationWarning) - self._streaming_callback = stack_context.wrap(streaming_callback) - try: - self._try_inline_read() - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_into(self, buf, callback=None, partial=False): - """Asynchronously read a number of bytes. - - ``buf`` must be a writable buffer into which data will be read. - If a callback is given, it will be run with the number of read - bytes as an argument; if not, this method returns a `.Future`. - - If ``partial`` is true, the callback is run as soon as any bytes - have been read. Otherwise, it is run when the ``buf`` has been - entirely filled with read data. - - .. versionadded:: 5.0 - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. Use the returned `.Future` instead. - - """ - future = self._set_read_callback(callback) - - # First copy data already in read buffer - available_bytes = self._read_buffer_size - n = len(buf) - if available_bytes >= n: - end = self._read_buffer_pos + n - buf[:] = memoryview(self._read_buffer)[self._read_buffer_pos:end] - del self._read_buffer[:end] - self._after_user_read_buffer = self._read_buffer - elif available_bytes > 0: - buf[:available_bytes] = memoryview(self._read_buffer)[self._read_buffer_pos:] - - # Set up the supplied buffer as our temporary read buffer. - # The original (if it had any data remaining) has been - # saved for later. - self._user_read_buffer = True - self._read_buffer = buf - self._read_buffer_pos = 0 - self._read_buffer_size = available_bytes - self._read_bytes = n - self._read_partial = partial - - try: - self._try_inline_read() - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def read_until_close(self, callback=None, streaming_callback=None): - """Asynchronously reads all data from the socket until it is closed. - - If a ``streaming_callback`` is given, it will be called with chunks - of data as they become available, and the final result will be empty. - Otherwise, the result is all the data that was read. - If a callback is given, it will be run with the data as an argument; - if not, this method returns a `.Future`. - - Note that if a ``streaming_callback`` is used, data will be - read from the socket as quickly as it becomes available; there - is no way to apply backpressure or cancel the reads. If flow - control or cancellation are desired, use a loop with - `read_bytes(partial=True) <.read_bytes>` instead. - - .. versionchanged:: 4.0 - The callback argument is now optional and a `.Future` will - be returned if it is omitted. - - .. deprecated:: 5.1 - - The ``callback`` and ``streaming_callback`` arguments are - deprecated and will be removed in Tornado 6.0. Use the - returned `.Future` (and `read_bytes` with ``partial=True`` - for ``streaming_callback``) instead. - - """ - future = self._set_read_callback(callback) - if streaming_callback is not None: - warnings.warn("streaming_callback is deprecated, use read_bytes(partial=True) instead", - DeprecationWarning) - self._streaming_callback = stack_context.wrap(streaming_callback) - if self.closed(): - if self._streaming_callback is not None: - self._run_read_callback(self._read_buffer_size, True) - self._run_read_callback(self._read_buffer_size, False) - return future - self._read_until_close = True - try: - self._try_inline_read() - except: - if future is not None: - future.add_done_callback(lambda f: f.exception()) - raise - return future - - def write(self, data, callback=None): - """Asynchronously write the given data to this stream. - - If ``callback`` is given, we call it when all of the buffered write - data has been successfully written to the stream. If there was - previously buffered write data and an old write callback, that - callback is simply overwritten with this new callback. - - If no ``callback`` is given, this method returns a `.Future` that - resolves (with a result of ``None``) when the write has been - completed. - - The ``data`` argument may be of type `bytes` or `memoryview`. - - .. versionchanged:: 4.0 - Now returns a `.Future` if no callback is given. - - .. versionchanged:: 4.5 - Added support for `memoryview` arguments. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. Use the returned `.Future` instead. - - """ - self._check_closed() - if data: - if (self.max_write_buffer_size is not None and - len(self._write_buffer) + len(data) > self.max_write_buffer_size): - raise StreamBufferFullError("Reached maximum write buffer size") - self._write_buffer.append(data) - self._total_write_index += len(data) - if callback is not None: - warnings.warn("callback argument is deprecated, use returned Future instead", - DeprecationWarning) - self._write_callback = stack_context.wrap(callback) - future = None - else: - future = Future() - future.add_done_callback(lambda f: f.exception()) - self._write_futures.append((self._total_write_index, future)) - if not self._connecting: - self._handle_write() - if self._write_buffer: - self._add_io_state(self.io_loop.WRITE) - self._maybe_add_error_listener() - return future - - def set_close_callback(self, callback): - """Call the given callback when the stream is closed. - - This mostly is not necessary for applications that use the - `.Future` interface; all outstanding ``Futures`` will resolve - with a `StreamClosedError` when the stream is closed. However, - it is still useful as a way to signal that the stream has been - closed while no other read or write is in progress. - - Unlike other callback-based interfaces, ``set_close_callback`` - will not be removed in Tornado 6.0. - """ - self._close_callback = stack_context.wrap(callback) - self._maybe_add_error_listener() - - def close(self, exc_info=False): - """Close this stream. - - If ``exc_info`` is true, set the ``error`` attribute to the current - exception from `sys.exc_info` (or if ``exc_info`` is a tuple, - use that instead of `sys.exc_info`). - """ - if not self.closed(): - if exc_info: - if isinstance(exc_info, tuple): - self.error = exc_info[1] - elif isinstance(exc_info, BaseException): - self.error = exc_info - else: - exc_info = sys.exc_info() - if any(exc_info): - self.error = exc_info[1] - if self._read_until_close: - if (self._streaming_callback is not None and - self._read_buffer_size): - self._run_read_callback(self._read_buffer_size, True) - self._read_until_close = False - self._run_read_callback(self._read_buffer_size, False) - if self._state is not None: - self.io_loop.remove_handler(self.fileno()) - self._state = None - self.close_fd() - self._closed = True - self._maybe_run_close_callback() - - def _maybe_run_close_callback(self): - # If there are pending callbacks, don't run the close callback - # until they're done (see _maybe_add_error_handler) - if self.closed() and self._pending_callbacks == 0: - futures = [] - if self._read_future is not None: - futures.append(self._read_future) - self._read_future = None - futures += [future for _, future in self._write_futures] - self._write_futures.clear() - if self._connect_future is not None: - futures.append(self._connect_future) - self._connect_future = None - if self._ssl_connect_future is not None: - futures.append(self._ssl_connect_future) - self._ssl_connect_future = None - for future in futures: - future.set_exception(StreamClosedError(real_error=self.error)) - future.exception() - if self._close_callback is not None: - cb = self._close_callback - self._close_callback = None - self._run_callback(cb) - # Delete any unfinished callbacks to break up reference cycles. - self._read_callback = self._write_callback = None - # Clear the buffers so they can be cleared immediately even - # if the IOStream object is kept alive by a reference cycle. - # TODO: Clear the read buffer too; it currently breaks some tests. - self._write_buffer = None - - def reading(self): - """Returns true if we are currently reading from the stream.""" - return self._read_callback is not None or self._read_future is not None - - def writing(self): - """Returns true if we are currently writing to the stream.""" - return bool(self._write_buffer) - - def closed(self): - """Returns true if the stream has been closed.""" - return self._closed - - def set_nodelay(self, value): - """Sets the no-delay flag for this stream. - - By default, data written to TCP streams may be held for a time - to make the most efficient use of bandwidth (according to - Nagle's algorithm). The no-delay flag requests that data be - written as soon as possible, even if doing so would consume - additional bandwidth. - - This flag is currently defined only for TCP-based ``IOStreams``. - - .. versionadded:: 3.1 - """ - pass - - def _handle_events(self, fd, events): - if self.closed(): - gen_log.warning("Got events for closed stream %s", fd) - return - try: - if self._connecting: - # Most IOLoops will report a write failed connect - # with the WRITE event, but SelectIOLoop reports a - # READ as well so we must check for connecting before - # either. - self._handle_connect() - if self.closed(): - return - if events & self.io_loop.READ: - self._handle_read() - if self.closed(): - return - if events & self.io_loop.WRITE: - self._handle_write() - if self.closed(): - return - if events & self.io_loop.ERROR: - self.error = self.get_fd_error() - # We may have queued up a user callback in _handle_read or - # _handle_write, so don't close the IOStream until those - # callbacks have had a chance to run. - self.io_loop.add_callback(self.close) - return - state = self.io_loop.ERROR - if self.reading(): - state |= self.io_loop.READ - if self.writing(): - state |= self.io_loop.WRITE - if state == self.io_loop.ERROR and self._read_buffer_size == 0: - # If the connection is idle, listen for reads too so - # we can tell if the connection is closed. If there is - # data in the read buffer we won't run the close callback - # yet anyway, so we don't need to listen in this case. - state |= self.io_loop.READ - if state != self._state: - assert self._state is not None, \ - "shouldn't happen: _handle_events without self._state" - self._state = state - self.io_loop.update_handler(self.fileno(), self._state) - except UnsatisfiableReadError as e: - gen_log.info("Unsatisfiable read, closing connection: %s" % e) - self.close(exc_info=e) - except Exception as e: - gen_log.error("Uncaught exception, closing connection.", - exc_info=True) - self.close(exc_info=e) - raise - - def _run_callback(self, callback, *args): - def wrapper(): - self._pending_callbacks -= 1 - try: - return callback(*args) - except Exception as e: - app_log.error("Uncaught exception, closing connection.", - exc_info=True) - # Close the socket on an uncaught exception from a user callback - # (It would eventually get closed when the socket object is - # gc'd, but we don't want to rely on gc happening before we - # run out of file descriptors) - self.close(exc_info=e) - # Re-raise the exception so that IOLoop.handle_callback_exception - # can see it and log the error - raise - finally: - self._maybe_add_error_listener() - # We schedule callbacks to be run on the next IOLoop iteration - # rather than running them directly for several reasons: - # * Prevents unbounded stack growth when a callback calls an - # IOLoop operation that immediately runs another callback - # * Provides a predictable execution context for e.g. - # non-reentrant mutexes - # * Ensures that the try/except in wrapper() is run outside - # of the application's StackContexts - with stack_context.NullContext(): - # stack_context was already captured in callback, we don't need to - # capture it again for IOStream's wrapper. This is especially - # important if the callback was pre-wrapped before entry to - # IOStream (as in HTTPConnection._header_callback), as we could - # capture and leak the wrong context here. - self._pending_callbacks += 1 - self.io_loop.add_callback(wrapper) - - def _read_to_buffer_loop(self): - # This method is called from _handle_read and _try_inline_read. - try: - if self._read_bytes is not None: - target_bytes = self._read_bytes - elif self._read_max_bytes is not None: - target_bytes = self._read_max_bytes - elif self.reading(): - # For read_until without max_bytes, or - # read_until_close, read as much as we can before - # scanning for the delimiter. - target_bytes = None - else: - target_bytes = 0 - next_find_pos = 0 - # Pretend to have a pending callback so that an EOF in - # _read_to_buffer doesn't trigger an immediate close - # callback. At the end of this method we'll either - # establish a real pending callback via - # _read_from_buffer or run the close callback. - # - # We need two try statements here so that - # pending_callbacks is decremented before the `except` - # clause below (which calls `close` and does need to - # trigger the callback) - self._pending_callbacks += 1 - while not self.closed(): - # Read from the socket until we get EWOULDBLOCK or equivalent. - # SSL sockets do some internal buffering, and if the data is - # sitting in the SSL object's buffer select() and friends - # can't see it; the only way to find out if it's there is to - # try to read it. - if self._read_to_buffer() == 0: - break - - self._run_streaming_callback() - - # If we've read all the bytes we can use, break out of - # this loop. We can't just call read_from_buffer here - # because of subtle interactions with the - # pending_callback and error_listener mechanisms. - # - # If we've reached target_bytes, we know we're done. - if (target_bytes is not None and - self._read_buffer_size >= target_bytes): - break - - # Otherwise, we need to call the more expensive find_read_pos. - # It's inefficient to do this on every read, so instead - # do it on the first read and whenever the read buffer - # size has doubled. - if self._read_buffer_size >= next_find_pos: - pos = self._find_read_pos() - if pos is not None: - return pos - next_find_pos = self._read_buffer_size * 2 - return self._find_read_pos() - finally: - self._pending_callbacks -= 1 - - def _handle_read(self): - try: - pos = self._read_to_buffer_loop() - except UnsatisfiableReadError: - raise - except Exception as e: - gen_log.warning("error on read: %s" % e) - self.close(exc_info=e) - return - if pos is not None: - self._read_from_buffer(pos) - return - else: - self._maybe_run_close_callback() - - def _set_read_callback(self, callback): - assert self._read_callback is None, "Already reading" - assert self._read_future is None, "Already reading" - if callback is not None: - warnings.warn("callbacks are deprecated, use returned Future instead", - DeprecationWarning) - self._read_callback = stack_context.wrap(callback) - else: - self._read_future = Future() - return self._read_future - - def _run_read_callback(self, size, streaming): - if self._user_read_buffer: - self._read_buffer = self._after_user_read_buffer or bytearray() - self._after_user_read_buffer = None - self._read_buffer_pos = 0 - self._read_buffer_size = len(self._read_buffer) - self._user_read_buffer = False - result = size - else: - result = self._consume(size) - if streaming: - callback = self._streaming_callback - else: - callback = self._read_callback - self._read_callback = self._streaming_callback = None - if self._read_future is not None: - assert callback is None - future = self._read_future - self._read_future = None - - future.set_result(result) - if callback is not None: - assert (self._read_future is None) or streaming - self._run_callback(callback, result) - else: - # If we scheduled a callback, we will add the error listener - # afterwards. If we didn't, we have to do it now. - self._maybe_add_error_listener() - - def _try_inline_read(self): - """Attempt to complete the current read operation from buffered data. - - If the read can be completed without blocking, schedules the - read callback on the next IOLoop iteration; otherwise starts - listening for reads on the socket. - """ - # See if we've already got the data from a previous read - self._run_streaming_callback() - pos = self._find_read_pos() - if pos is not None: - self._read_from_buffer(pos) - return - self._check_closed() - try: - pos = self._read_to_buffer_loop() - except Exception: - # If there was an in _read_to_buffer, we called close() already, - # but couldn't run the close callback because of _pending_callbacks. - # Before we escape from this function, run the close callback if - # applicable. - self._maybe_run_close_callback() - raise - if pos is not None: - self._read_from_buffer(pos) - return - # We couldn't satisfy the read inline, so either close the stream - # or listen for new data. - if self.closed(): - self._maybe_run_close_callback() - else: - self._add_io_state(ioloop.IOLoop.READ) - - def _read_to_buffer(self): - """Reads from the socket and appends the result to the read buffer. - - Returns the number of bytes read. Returns 0 if there is nothing - to read (i.e. the read returns EWOULDBLOCK or equivalent). On - error closes the socket and raises an exception. - """ - try: - while True: - try: - if self._user_read_buffer: - buf = memoryview(self._read_buffer)[self._read_buffer_size:] - else: - buf = bytearray(self.read_chunk_size) - bytes_read = self.read_from_fd(buf) - except (socket.error, IOError, OSError) as e: - if errno_from_exception(e) == errno.EINTR: - continue - # ssl.SSLError is a subclass of socket.error - if self._is_connreset(e): - # Treat ECONNRESET as a connection close rather than - # an error to minimize log spam (the exception will - # be available on self.error for apps that care). - self.close(exc_info=e) - return - self.close(exc_info=e) - raise - break - if bytes_read is None: - return 0 - elif bytes_read == 0: - self.close() - return 0 - if not self._user_read_buffer: - self._read_buffer += memoryview(buf)[:bytes_read] - self._read_buffer_size += bytes_read - finally: - # Break the reference to buf so we don't waste a chunk's worth of - # memory in case an exception hangs on to our stack frame. - buf = None - if self._read_buffer_size > self.max_buffer_size: - gen_log.error("Reached maximum read buffer size") - self.close() - raise StreamBufferFullError("Reached maximum read buffer size") - return bytes_read - - def _run_streaming_callback(self): - if self._streaming_callback is not None and self._read_buffer_size: - bytes_to_consume = self._read_buffer_size - if self._read_bytes is not None: - bytes_to_consume = min(self._read_bytes, bytes_to_consume) - self._read_bytes -= bytes_to_consume - self._run_read_callback(bytes_to_consume, True) - - def _read_from_buffer(self, pos): - """Attempts to complete the currently-pending read from the buffer. - - The argument is either a position in the read buffer or None, - as returned by _find_read_pos. - """ - self._read_bytes = self._read_delimiter = self._read_regex = None - self._read_partial = False - self._run_read_callback(pos, False) - - def _find_read_pos(self): - """Attempts to find a position in the read buffer that satisfies - the currently-pending read. - - Returns a position in the buffer if the current read can be satisfied, - or None if it cannot. - """ - if (self._read_bytes is not None and - (self._read_buffer_size >= self._read_bytes or - (self._read_partial and self._read_buffer_size > 0))): - num_bytes = min(self._read_bytes, self._read_buffer_size) - return num_bytes - elif self._read_delimiter is not None: - # Multi-byte delimiters (e.g. '\r\n') may straddle two - # chunks in the read buffer, so we can't easily find them - # without collapsing the buffer. However, since protocols - # using delimited reads (as opposed to reads of a known - # length) tend to be "line" oriented, the delimiter is likely - # to be in the first few chunks. Merge the buffer gradually - # since large merges are relatively expensive and get undone in - # _consume(). - if self._read_buffer: - loc = self._read_buffer.find(self._read_delimiter, - self._read_buffer_pos) - if loc != -1: - loc -= self._read_buffer_pos - delimiter_len = len(self._read_delimiter) - self._check_max_bytes(self._read_delimiter, - loc + delimiter_len) - return loc + delimiter_len - self._check_max_bytes(self._read_delimiter, - self._read_buffer_size) - elif self._read_regex is not None: - if self._read_buffer: - m = self._read_regex.search(self._read_buffer, - self._read_buffer_pos) - if m is not None: - loc = m.end() - self._read_buffer_pos - self._check_max_bytes(self._read_regex, loc) - return loc - self._check_max_bytes(self._read_regex, self._read_buffer_size) - return None - - def _check_max_bytes(self, delimiter, size): - if (self._read_max_bytes is not None and - size > self._read_max_bytes): - raise UnsatisfiableReadError( - "delimiter %r not found within %d bytes" % ( - delimiter, self._read_max_bytes)) - - def _handle_write(self): - while True: - size = len(self._write_buffer) - if not size: - break - assert size > 0 - try: - if _WINDOWS: - # On windows, socket.send blows up if given a - # write buffer that's too large, instead of just - # returning the number of bytes it was able to - # process. Therefore we must not call socket.send - # with more than 128KB at a time. - size = 128 * 1024 - - num_bytes = self.write_to_fd(self._write_buffer.peek(size)) - if num_bytes == 0: - break - self._write_buffer.advance(num_bytes) - self._total_write_done_index += num_bytes - except (socket.error, IOError, OSError) as e: - if e.args[0] in _ERRNO_WOULDBLOCK: - break - else: - if not self._is_connreset(e): - # Broken pipe errors are usually caused by connection - # reset, and its better to not log EPIPE errors to - # minimize log spam - gen_log.warning("Write error on %s: %s", - self.fileno(), e) - self.close(exc_info=e) - return - - while self._write_futures: - index, future = self._write_futures[0] - if index > self._total_write_done_index: - break - self._write_futures.popleft() - future.set_result(None) - - if not len(self._write_buffer): - if self._write_callback: - callback = self._write_callback - self._write_callback = None - self._run_callback(callback) - - def _consume(self, loc): - # Consume loc bytes from the read buffer and return them - if loc == 0: - return b"" - assert loc <= self._read_buffer_size - # Slice the bytearray buffer into bytes, without intermediate copying - b = (memoryview(self._read_buffer) - [self._read_buffer_pos:self._read_buffer_pos + loc] - ).tobytes() - self._read_buffer_pos += loc - self._read_buffer_size -= loc - # Amortized O(1) shrink - # (this heuristic is implemented natively in Python 3.4+ - # but is replicated here for Python 2) - if self._read_buffer_pos > self._read_buffer_size: - del self._read_buffer[:self._read_buffer_pos] - self._read_buffer_pos = 0 - return b - - def _check_closed(self): - if self.closed(): - raise StreamClosedError(real_error=self.error) - - def _maybe_add_error_listener(self): - # This method is part of an optimization: to detect a connection that - # is closed when we're not actively reading or writing, we must listen - # for read events. However, it is inefficient to do this when the - # connection is first established because we are going to read or write - # immediately anyway. Instead, we insert checks at various times to - # see if the connection is idle and add the read listener then. - if self._pending_callbacks != 0: - return - if self._state is None or self._state == ioloop.IOLoop.ERROR: - if self.closed(): - self._maybe_run_close_callback() - elif (self._read_buffer_size == 0 and - self._close_callback is not None): - self._add_io_state(ioloop.IOLoop.READ) - - def _add_io_state(self, state): - """Adds `state` (IOLoop.{READ,WRITE} flags) to our event handler. - - Implementation notes: Reads and writes have a fast path and a - slow path. The fast path reads synchronously from socket - buffers, while the slow path uses `_add_io_state` to schedule - an IOLoop callback. Note that in both cases, the callback is - run asynchronously with `_run_callback`. - - To detect closed connections, we must have called - `_add_io_state` at some point, but we want to delay this as - much as possible so we don't have to set an `IOLoop.ERROR` - listener that will be overwritten by the next slow-path - operation. As long as there are callbacks scheduled for - fast-path ops, those callbacks may do more reads. - If a sequence of fast-path ops do not end in a slow-path op, - (e.g. for an @asynchronous long-poll request), we must add - the error handler. This is done in `_run_callback` and `write` - (since the write callback is optional so we can have a - fast-path write with no `_run_callback`) - """ - if self.closed(): - # connection has been closed, so there can be no future events - return - if self._state is None: - self._state = ioloop.IOLoop.ERROR | state - with stack_context.NullContext(): - self.io_loop.add_handler( - self.fileno(), self._handle_events, self._state) - elif not self._state & state: - self._state = self._state | state - self.io_loop.update_handler(self.fileno(), self._state) - - def _is_connreset(self, exc): - """Return true if exc is ECONNRESET or equivalent. - - May be overridden in subclasses. - """ - return (isinstance(exc, (socket.error, IOError)) and - errno_from_exception(exc) in _ERRNO_CONNRESET) - - -class IOStream(BaseIOStream): - r"""Socket-based `IOStream` implementation. - - This class supports the read and write methods from `BaseIOStream` - plus a `connect` method. - - The ``socket`` parameter may either be connected or unconnected. - For server operations the socket is the result of calling - `socket.accept <socket.socket.accept>`. For client operations the - socket is created with `socket.socket`, and may either be - connected before passing it to the `IOStream` or connected with - `IOStream.connect`. - - A very simple (and broken) HTTP client using this class: - - .. testcode:: - - import tornado.ioloop - import tornado.iostream - import socket - - async def main(): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - stream = tornado.iostream.IOStream(s) - await stream.connect(("friendfeed.com", 80)) - await stream.write(b"GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") - header_data = await stream.read_until(b"\r\n\r\n") - headers = {} - for line in header_data.split(b"\r\n"): - parts = line.split(b":") - if len(parts) == 2: - headers[parts[0].strip()] = parts[1].strip() - body_data = await stream.read_bytes(int(headers[b"Content-Length"])) - print(body_data) - stream.close() - - if __name__ == '__main__': - tornado.ioloop.IOLoop.current().run_sync(main) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - stream = tornado.iostream.IOStream(s) - stream.connect(("friendfeed.com", 80), send_request) - tornado.ioloop.IOLoop.current().start() - - .. testoutput:: - :hide: - - """ - def __init__(self, socket, *args, **kwargs): - self.socket = socket - self.socket.setblocking(False) - super(IOStream, self).__init__(*args, **kwargs) - - def fileno(self): - return self.socket - - def close_fd(self): - self.socket.close() - self.socket = None - - def get_fd_error(self): - errno = self.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_ERROR) - return socket.error(errno, os.strerror(errno)) - - def read_from_fd(self, buf): - try: - return self.socket.recv_into(buf) - except socket.error as e: - if e.args[0] in _ERRNO_WOULDBLOCK: - return None - else: - raise - finally: - buf = None - - def write_to_fd(self, data): - try: - return self.socket.send(data) - finally: - # Avoid keeping to data, which can be a memoryview. - # See https://github.com/tornadoweb/tornado/pull/2008 - del data - - def connect(self, address, callback=None, server_hostname=None): - """Connects the socket to a remote address without blocking. - - May only be called if the socket passed to the constructor was - not previously connected. The address parameter is in the - same format as for `socket.connect <socket.socket.connect>` for - the type of socket passed to the IOStream constructor, - e.g. an ``(ip, port)`` tuple. Hostnames are accepted here, - but will be resolved synchronously and block the IOLoop. - If you have a hostname instead of an IP address, the `.TCPClient` - class is recommended instead of calling this method directly. - `.TCPClient` will do asynchronous DNS resolution and handle - both IPv4 and IPv6. - - If ``callback`` is specified, it will be called with no - arguments when the connection is completed; if not this method - returns a `.Future` (whose result after a successful - connection will be the stream itself). - - In SSL mode, the ``server_hostname`` parameter will be used - for certificate validation (unless disabled in the - ``ssl_options``) and SNI (if supported; requires Python - 2.7.9+). - - Note that it is safe to call `IOStream.write - <BaseIOStream.write>` while the connection is pending, in - which case the data will be written as soon as the connection - is ready. Calling `IOStream` read methods before the socket is - connected works on some platforms but is non-portable. - - .. versionchanged:: 4.0 - If no callback is given, returns a `.Future`. - - .. versionchanged:: 4.2 - SSL certificates are validated by default; pass - ``ssl_options=dict(cert_reqs=ssl.CERT_NONE)`` or a - suitably-configured `ssl.SSLContext` to the - `SSLIOStream` constructor to disable. - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. Use the returned `.Future` instead. - - """ - self._connecting = True - if callback is not None: - warnings.warn("callback argument is deprecated, use returned Future instead", - DeprecationWarning) - self._connect_callback = stack_context.wrap(callback) - future = None - else: - future = self._connect_future = Future() - try: - self.socket.connect(address) - except socket.error as e: - # In non-blocking mode we expect connect() to raise an - # exception with EINPROGRESS or EWOULDBLOCK. - # - # On freebsd, other errors such as ECONNREFUSED may be - # returned immediately when attempting to connect to - # localhost, so handle them the same way as an error - # reported later in _handle_connect. - if (errno_from_exception(e) not in _ERRNO_INPROGRESS and - errno_from_exception(e) not in _ERRNO_WOULDBLOCK): - if future is None: - gen_log.warning("Connect error on fd %s: %s", - self.socket.fileno(), e) - self.close(exc_info=e) - return future - self._add_io_state(self.io_loop.WRITE) - return future - - def start_tls(self, server_side, ssl_options=None, server_hostname=None): - """Convert this `IOStream` to an `SSLIOStream`. - - This enables protocols that begin in clear-text mode and - switch to SSL after some initial negotiation (such as the - ``STARTTLS`` extension to SMTP and IMAP). - - This method cannot be used if there are outstanding reads - or writes on the stream, or if there is any data in the - IOStream's buffer (data in the operating system's socket - buffer is allowed). This means it must generally be used - immediately after reading or writing the last clear-text - data. It can also be used immediately after connecting, - before any reads or writes. - - The ``ssl_options`` argument may be either an `ssl.SSLContext` - object or a dictionary of keyword arguments for the - `ssl.wrap_socket` function. The ``server_hostname`` argument - will be used for certificate validation unless disabled - in the ``ssl_options``. - - This method returns a `.Future` whose result is the new - `SSLIOStream`. After this method has been called, - any other operation on the original stream is undefined. - - If a close callback is defined on this stream, it will be - transferred to the new stream. - - .. versionadded:: 4.0 - - .. versionchanged:: 4.2 - SSL certificates are validated by default; pass - ``ssl_options=dict(cert_reqs=ssl.CERT_NONE)`` or a - suitably-configured `ssl.SSLContext` to disable. - """ - if (self._read_callback or self._read_future or - self._write_callback or self._write_futures or - self._connect_callback or self._connect_future or - self._pending_callbacks or self._closed or - self._read_buffer or self._write_buffer): - raise ValueError("IOStream is not idle; cannot convert to SSL") - if ssl_options is None: - if server_side: - ssl_options = _server_ssl_defaults - else: - ssl_options = _client_ssl_defaults - - socket = self.socket - self.io_loop.remove_handler(socket) - self.socket = None - socket = ssl_wrap_socket(socket, ssl_options, - server_hostname=server_hostname, - server_side=server_side, - do_handshake_on_connect=False) - orig_close_callback = self._close_callback - self._close_callback = None - - future = Future() - ssl_stream = SSLIOStream(socket, ssl_options=ssl_options) - # Wrap the original close callback so we can fail our Future as well. - # If we had an "unwrap" counterpart to this method we would need - # to restore the original callback after our Future resolves - # so that repeated wrap/unwrap calls don't build up layers. - - def close_callback(): - if not future.done(): - # Note that unlike most Futures returned by IOStream, - # this one passes the underlying error through directly - # instead of wrapping everything in a StreamClosedError - # with a real_error attribute. This is because once the - # connection is established it's more helpful to raise - # the SSLError directly than to hide it behind a - # StreamClosedError (and the client is expecting SSL - # issues rather than network issues since this method is - # named start_tls). - future.set_exception(ssl_stream.error or StreamClosedError()) - if orig_close_callback is not None: - orig_close_callback() - ssl_stream.set_close_callback(close_callback) - ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream) - ssl_stream.max_buffer_size = self.max_buffer_size - ssl_stream.read_chunk_size = self.read_chunk_size - return future - - def _handle_connect(self): - try: - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - except socket.error as e: - # Hurd doesn't allow SO_ERROR for loopback sockets because all - # errors for such sockets are reported synchronously. - if errno_from_exception(e) == errno.ENOPROTOOPT: - err = 0 - if err != 0: - self.error = socket.error(err, os.strerror(err)) - # IOLoop implementations may vary: some of them return - # an error state before the socket becomes writable, so - # in that case a connection failure would be handled by the - # error path in _handle_events instead of here. - if self._connect_future is None: - gen_log.warning("Connect error on fd %s: %s", - self.socket.fileno(), errno.errorcode[err]) - self.close() - return - if self._connect_callback is not None: - callback = self._connect_callback - self._connect_callback = None - self._run_callback(callback) - if self._connect_future is not None: - future = self._connect_future - self._connect_future = None - future.set_result(self) - self._connecting = False - - def set_nodelay(self, value): - if (self.socket is not None and - self.socket.family in (socket.AF_INET, socket.AF_INET6)): - try: - self.socket.setsockopt(socket.IPPROTO_TCP, - socket.TCP_NODELAY, 1 if value else 0) - except socket.error as e: - # Sometimes setsockopt will fail if the socket is closed - # at the wrong time. This can happen with HTTPServer - # resetting the value to false between requests. - if e.errno != errno.EINVAL and not self._is_connreset(e): - raise - - -class SSLIOStream(IOStream): - """A utility class to write to and read from a non-blocking SSL socket. - - If the socket passed to the constructor is already connected, - it should be wrapped with:: - - ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) - - before constructing the `SSLIOStream`. Unconnected sockets will be - wrapped when `IOStream.connect` is finished. - """ - def __init__(self, *args, **kwargs): - """The ``ssl_options`` keyword argument may either be an - `ssl.SSLContext` object or a dictionary of keywords arguments - for `ssl.wrap_socket` - """ - self._ssl_options = kwargs.pop('ssl_options', _client_ssl_defaults) - super(SSLIOStream, self).__init__(*args, **kwargs) - self._ssl_accepting = True - self._handshake_reading = False - self._handshake_writing = False - self._ssl_connect_callback = None - self._server_hostname = None - - # If the socket is already connected, attempt to start the handshake. - try: - self.socket.getpeername() - except socket.error: - pass - else: - # Indirectly start the handshake, which will run on the next - # IOLoop iteration and then the real IO state will be set in - # _handle_events. - self._add_io_state(self.io_loop.WRITE) - - def reading(self): - return self._handshake_reading or super(SSLIOStream, self).reading() - - def writing(self): - return self._handshake_writing or super(SSLIOStream, self).writing() - - def _do_ssl_handshake(self): - # Based on code from test_ssl.py in the python stdlib - try: - self._handshake_reading = False - self._handshake_writing = False - self.socket.do_handshake() - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_WANT_READ: - self._handshake_reading = True - return - elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: - self._handshake_writing = True - return - elif err.args[0] in (ssl.SSL_ERROR_EOF, - ssl.SSL_ERROR_ZERO_RETURN): - return self.close(exc_info=err) - elif err.args[0] == ssl.SSL_ERROR_SSL: - try: - peer = self.socket.getpeername() - except Exception: - peer = '(not connected)' - gen_log.warning("SSL Error on %s %s: %s", - self.socket.fileno(), peer, err) - return self.close(exc_info=err) - raise - except socket.error as err: - # Some port scans (e.g. nmap in -sT mode) have been known - # to cause do_handshake to raise EBADF and ENOTCONN, so make - # those errors quiet as well. - # https://groups.google.com/forum/?fromgroups#!topic/python-tornado/ApucKJat1_0 - if (self._is_connreset(err) or - err.args[0] in (errno.EBADF, errno.ENOTCONN)): - return self.close(exc_info=err) - raise - except AttributeError as err: - # On Linux, if the connection was reset before the call to - # wrap_socket, do_handshake will fail with an - # AttributeError. - return self.close(exc_info=err) - else: - self._ssl_accepting = False - if not self._verify_cert(self.socket.getpeercert()): - self.close() - return - self._run_ssl_connect_callback() - - def _run_ssl_connect_callback(self): - if self._ssl_connect_callback is not None: - callback = self._ssl_connect_callback - self._ssl_connect_callback = None - self._run_callback(callback) - if self._ssl_connect_future is not None: - future = self._ssl_connect_future - self._ssl_connect_future = None - future.set_result(self) - - def _verify_cert(self, peercert): - """Returns True if peercert is valid according to the configured - validation mode and hostname. - - The ssl handshake already tested the certificate for a valid - CA signature; the only thing that remains is to check - the hostname. - """ - if isinstance(self._ssl_options, dict): - verify_mode = self._ssl_options.get('cert_reqs', ssl.CERT_NONE) - elif isinstance(self._ssl_options, ssl.SSLContext): - verify_mode = self._ssl_options.verify_mode - assert verify_mode in (ssl.CERT_NONE, ssl.CERT_REQUIRED, ssl.CERT_OPTIONAL) - if verify_mode == ssl.CERT_NONE or self._server_hostname is None: - return True - cert = self.socket.getpeercert() - if cert is None and verify_mode == ssl.CERT_REQUIRED: - gen_log.warning("No SSL certificate given") - return False - try: - ssl.match_hostname(peercert, self._server_hostname) - except ssl.CertificateError as e: - gen_log.warning("Invalid SSL certificate: %s" % e) - return False - else: - return True - - def _handle_read(self): - if self._ssl_accepting: - self._do_ssl_handshake() - return - super(SSLIOStream, self)._handle_read() - - def _handle_write(self): - if self._ssl_accepting: - self._do_ssl_handshake() - return - super(SSLIOStream, self)._handle_write() - - def connect(self, address, callback=None, server_hostname=None): - self._server_hostname = server_hostname - # Ignore the result of connect(). If it fails, - # wait_for_handshake will raise an error too. This is - # necessary for the old semantics of the connect callback - # (which takes no arguments). In 6.0 this can be refactored to - # be a regular coroutine. - fut = super(SSLIOStream, self).connect(address) - fut.add_done_callback(lambda f: f.exception()) - return self.wait_for_handshake(callback) - - def _handle_connect(self): - # Call the superclass method to check for errors. - super(SSLIOStream, self)._handle_connect() - if self.closed(): - return - # When the connection is complete, wrap the socket for SSL - # traffic. Note that we do this by overriding _handle_connect - # instead of by passing a callback to super().connect because - # user callbacks are enqueued asynchronously on the IOLoop, - # but since _handle_events calls _handle_connect immediately - # followed by _handle_write we need this to be synchronous. - # - # The IOLoop will get confused if we swap out self.socket while the - # fd is registered, so remove it now and re-register after - # wrap_socket(). - self.io_loop.remove_handler(self.socket) - old_state = self._state - self._state = None - self.socket = ssl_wrap_socket(self.socket, self._ssl_options, - server_hostname=self._server_hostname, - do_handshake_on_connect=False) - self._add_io_state(old_state) - - def wait_for_handshake(self, callback=None): - """Wait for the initial SSL handshake to complete. - - If a ``callback`` is given, it will be called with no - arguments once the handshake is complete; otherwise this - method returns a `.Future` which will resolve to the - stream itself after the handshake is complete. - - Once the handshake is complete, information such as - the peer's certificate and NPN/ALPN selections may be - accessed on ``self.socket``. - - This method is intended for use on server-side streams - or after using `IOStream.start_tls`; it should not be used - with `IOStream.connect` (which already waits for the - handshake to complete). It may only be called once per stream. - - .. versionadded:: 4.2 - - .. deprecated:: 5.1 - - The ``callback`` argument is deprecated and will be removed - in Tornado 6.0. Use the returned `.Future` instead. - - """ - if (self._ssl_connect_callback is not None or - self._ssl_connect_future is not None): - raise RuntimeError("Already waiting") - if callback is not None: - warnings.warn("callback argument is deprecated, use returned Future instead", - DeprecationWarning) - self._ssl_connect_callback = stack_context.wrap(callback) - future = None - else: - future = self._ssl_connect_future = Future() - if not self._ssl_accepting: - self._run_ssl_connect_callback() - return future - - def write_to_fd(self, data): - try: - return self.socket.send(data) - except ssl.SSLError as e: - if e.args[0] == ssl.SSL_ERROR_WANT_WRITE: - # In Python 3.5+, SSLSocket.send raises a WANT_WRITE error if - # the socket is not writeable; we need to transform this into - # an EWOULDBLOCK socket.error or a zero return value, - # either of which will be recognized by the caller of this - # method. Prior to Python 3.5, an unwriteable socket would - # simply return 0 bytes written. - return 0 - raise - finally: - # Avoid keeping to data, which can be a memoryview. - # See https://github.com/tornadoweb/tornado/pull/2008 - del data - - def read_from_fd(self, buf): - try: - if self._ssl_accepting: - # If the handshake hasn't finished yet, there can't be anything - # to read (attempting to read may or may not raise an exception - # depending on the SSL version) - return None - try: - return self.socket.recv_into(buf) - except ssl.SSLError as e: - # SSLError is a subclass of socket.error, so this except - # block must come first. - if e.args[0] == ssl.SSL_ERROR_WANT_READ: - return None - else: - raise - except socket.error as e: - if e.args[0] in _ERRNO_WOULDBLOCK: - return None - else: - raise - finally: - buf = None - - def _is_connreset(self, e): - if isinstance(e, ssl.SSLError) and e.args[0] == ssl.SSL_ERROR_EOF: - return True - return super(SSLIOStream, self)._is_connreset(e) - - -class PipeIOStream(BaseIOStream): - """Pipe-based `IOStream` implementation. - - The constructor takes an integer file descriptor (such as one returned - by `os.pipe`) rather than an open file object. Pipes are generally - one-way, so a `PipeIOStream` can be used for reading or writing but not - both. - """ - def __init__(self, fd, *args, **kwargs): - self.fd = fd - self._fio = io.FileIO(self.fd, "r+") - _set_nonblocking(fd) - super(PipeIOStream, self).__init__(*args, **kwargs) - - def fileno(self): - return self.fd - - def close_fd(self): - self._fio.close() - - def write_to_fd(self, data): - try: - return os.write(self.fd, data) - finally: - # Avoid keeping to data, which can be a memoryview. - # See https://github.com/tornadoweb/tornado/pull/2008 - del data - - def read_from_fd(self, buf): - try: - return self._fio.readinto(buf) - except (IOError, OSError) as e: - if errno_from_exception(e) == errno.EBADF: - # If the writing half of a pipe is closed, select will - # report it as readable but reads will fail with EBADF. - self.close(exc_info=e) - return None - else: - raise - finally: - buf = None - - -def doctests(): - import doctest - return doctest.DocTestSuite() diff --git a/lib/tornado/locale.py b/lib/tornado/locale.py deleted file mode 100755 index d45172f3..00000000 --- a/lib/tornado/locale.py +++ /dev/null @@ -1,521 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Translation methods for generating localized strings. - -To load a locale and generate a translated string:: - - user_locale = tornado.locale.get("es_LA") - print(user_locale.translate("Sign out")) - -`tornado.locale.get()` returns the closest matching locale, not necessarily the -specific locale you requested. You can support pluralization with -additional arguments to `~Locale.translate()`, e.g.:: - - people = [...] - message = user_locale.translate( - "%(list)s is online", "%(list)s are online", len(people)) - print(message % {"list": user_locale.list(people)}) - -The first string is chosen if ``len(people) == 1``, otherwise the second -string is chosen. - -Applications should call one of `load_translations` (which uses a simple -CSV format) or `load_gettext_translations` (which uses the ``.mo`` format -supported by `gettext` and related tools). If neither method is called, -the `Locale.translate` method will simply return the original string. -""" - -from __future__ import absolute_import, division, print_function - -import codecs -import csv -import datetime -from io import BytesIO -import numbers -import os -import re - -from tornado import escape -from tornado.log import gen_log -from tornado.util import PY3 - -from tornado._locale_data import LOCALE_NAMES - -_default_locale = "en_US" -_translations = {} # type: dict -_supported_locales = frozenset([_default_locale]) -_use_gettext = False -CONTEXT_SEPARATOR = "\x04" - - -def get(*locale_codes): - """Returns the closest match for the given locale codes. - - We iterate over all given locale codes in order. If we have a tight - or a loose match for the code (e.g., "en" for "en_US"), we return - the locale. Otherwise we move to the next code in the list. - - By default we return ``en_US`` if no translations are found for any of - the specified locales. You can change the default locale with - `set_default_locale()`. - """ - return Locale.get_closest(*locale_codes) - - -def set_default_locale(code): - """Sets the default locale. - - The default locale is assumed to be the language used for all strings - in the system. The translations loaded from disk are mappings from - the default locale to the destination locale. Consequently, you don't - need to create a translation file for the default locale. - """ - global _default_locale - global _supported_locales - _default_locale = code - _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) - - -def load_translations(directory, encoding=None): - """Loads translations from CSV files in a directory. - - Translations are strings with optional Python-style named placeholders - (e.g., ``My name is %(name)s``) and their associated translations. - - The directory should have translation files of the form ``LOCALE.csv``, - e.g. ``es_GT.csv``. The CSV files should have two or three columns: string, - translation, and an optional plural indicator. Plural indicators should - be one of "plural" or "singular". A given string can have both singular - and plural forms. For example ``%(name)s liked this`` may have a - different verb conjugation depending on whether %(name)s is one - name or a list of names. There should be two rows in the CSV file for - that string, one with plural indicator "singular", and one "plural". - For strings with no verbs that would change on translation, simply - use "unknown" or the empty string (or don't include the column at all). - - The file is read using the `csv` module in the default "excel" dialect. - In this format there should not be spaces after the commas. - - If no ``encoding`` parameter is given, the encoding will be - detected automatically (among UTF-8 and UTF-16) if the file - contains a byte-order marker (BOM), defaulting to UTF-8 if no BOM - is present. - - Example translation ``es_LA.csv``:: - - "I love you","Te amo" - "%(name)s liked this","A %(name)s les gustó esto","plural" - "%(name)s liked this","A %(name)s le gustó esto","singular" - - .. versionchanged:: 4.3 - Added ``encoding`` parameter. Added support for BOM-based encoding - detection, UTF-16, and UTF-8-with-BOM. - """ - global _translations - global _supported_locales - _translations = {} - for path in os.listdir(directory): - if not path.endswith(".csv"): - continue - locale, extension = path.split(".") - if not re.match("[a-z]+(_[A-Z]+)?$", locale): - gen_log.error("Unrecognized locale %r (path: %s)", locale, - os.path.join(directory, path)) - continue - full_path = os.path.join(directory, path) - if encoding is None: - # Try to autodetect encoding based on the BOM. - with open(full_path, 'rb') as f: - data = f.read(len(codecs.BOM_UTF16_LE)) - if data in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): - encoding = 'utf-16' - else: - # utf-8-sig is "utf-8 with optional BOM". It's discouraged - # in most cases but is common with CSV files because Excel - # cannot read utf-8 files without a BOM. - encoding = 'utf-8-sig' - if PY3: - # python 3: csv.reader requires a file open in text mode. - # Force utf8 to avoid dependence on $LANG environment variable. - f = open(full_path, "r", encoding=encoding) - else: - # python 2: csv can only handle byte strings (in ascii-compatible - # encodings), which we decode below. Transcode everything into - # utf8 before passing it to csv.reader. - f = BytesIO() - with codecs.open(full_path, "r", encoding=encoding) as infile: - f.write(escape.utf8(infile.read())) - f.seek(0) - _translations[locale] = {} - for i, row in enumerate(csv.reader(f)): - if not row or len(row) < 2: - continue - row = [escape.to_unicode(c).strip() for c in row] - english, translation = row[:2] - if len(row) > 2: - plural = row[2] or "unknown" - else: - plural = "unknown" - if plural not in ("plural", "singular", "unknown"): - gen_log.error("Unrecognized plural indicator %r in %s line %d", - plural, path, i + 1) - continue - _translations[locale].setdefault(plural, {})[english] = translation - f.close() - _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) - gen_log.debug("Supported locales: %s", sorted(_supported_locales)) - - -def load_gettext_translations(directory, domain): - """Loads translations from `gettext`'s locale tree - - Locale tree is similar to system's ``/usr/share/locale``, like:: - - {directory}/{lang}/LC_MESSAGES/{domain}.mo - - Three steps are required to have your app translated: - - 1. Generate POT translation file:: - - xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc - - 2. Merge against existing POT file:: - - msgmerge old.po mydomain.po > new.po - - 3. Compile:: - - msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo - """ - import gettext - global _translations - global _supported_locales - global _use_gettext - _translations = {} - for lang in os.listdir(directory): - if lang.startswith('.'): - continue # skip .svn, etc - if os.path.isfile(os.path.join(directory, lang)): - continue - try: - os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo")) - _translations[lang] = gettext.translation(domain, directory, - languages=[lang]) - except Exception as e: - gen_log.error("Cannot load translation for '%s': %s", lang, str(e)) - continue - _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) - _use_gettext = True - gen_log.debug("Supported locales: %s", sorted(_supported_locales)) - - -def get_supported_locales(): - """Returns a list of all the supported locale codes.""" - return _supported_locales - - -class Locale(object): - """Object representing a locale. - - After calling one of `load_translations` or `load_gettext_translations`, - call `get` or `get_closest` to get a Locale object. - """ - @classmethod - def get_closest(cls, *locale_codes): - """Returns the closest match for the given locale code.""" - for code in locale_codes: - if not code: - continue - code = code.replace("-", "_") - parts = code.split("_") - if len(parts) > 2: - continue - elif len(parts) == 2: - code = parts[0].lower() + "_" + parts[1].upper() - if code in _supported_locales: - return cls.get(code) - if parts[0].lower() in _supported_locales: - return cls.get(parts[0].lower()) - return cls.get(_default_locale) - - @classmethod - def get(cls, code): - """Returns the Locale for the given locale code. - - If it is not supported, we raise an exception. - """ - if not hasattr(cls, "_cache"): - cls._cache = {} - if code not in cls._cache: - assert code in _supported_locales - translations = _translations.get(code, None) - if translations is None: - locale = CSVLocale(code, {}) - elif _use_gettext: - locale = GettextLocale(code, translations) - else: - locale = CSVLocale(code, translations) - cls._cache[code] = locale - return cls._cache[code] - - def __init__(self, code, translations): - self.code = code - self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown") - self.rtl = False - for prefix in ["fa", "ar", "he"]: - if self.code.startswith(prefix): - self.rtl = True - break - self.translations = translations - - # Initialize strings for date formatting - _ = self.translate - self._months = [ - _("January"), _("February"), _("March"), _("April"), - _("May"), _("June"), _("July"), _("August"), - _("September"), _("October"), _("November"), _("December")] - self._weekdays = [ - _("Monday"), _("Tuesday"), _("Wednesday"), _("Thursday"), - _("Friday"), _("Saturday"), _("Sunday")] - - def translate(self, message, plural_message=None, count=None): - """Returns the translation for the given message for this locale. - - If ``plural_message`` is given, you must also provide - ``count``. We return ``plural_message`` when ``count != 1``, - and we return the singular form for the given message when - ``count == 1``. - """ - raise NotImplementedError() - - def pgettext(self, context, message, plural_message=None, count=None): - raise NotImplementedError() - - def format_date(self, date, gmt_offset=0, relative=True, shorter=False, - full_format=False): - """Formats the given date (which should be GMT). - - By default, we return a relative time (e.g., "2 minutes ago"). You - can return an absolute date string with ``relative=False``. - - You can force a full format date ("July 10, 1980") with - ``full_format=True``. - - This method is primarily intended for dates in the past. - For dates in the future, we fall back to full format. - """ - if isinstance(date, numbers.Real): - date = datetime.datetime.utcfromtimestamp(date) - now = datetime.datetime.utcnow() - if date > now: - if relative and (date - now).seconds < 60: - # Due to click skew, things are some things slightly - # in the future. Round timestamps in the immediate - # future down to now in relative mode. - date = now - else: - # Otherwise, future dates always use the full format. - full_format = True - local_date = date - datetime.timedelta(minutes=gmt_offset) - local_now = now - datetime.timedelta(minutes=gmt_offset) - local_yesterday = local_now - datetime.timedelta(hours=24) - difference = now - date - seconds = difference.seconds - days = difference.days - - _ = self.translate - format = None - if not full_format: - if relative and days == 0: - if seconds < 50: - return _("1 second ago", "%(seconds)d seconds ago", - seconds) % {"seconds": seconds} - - if seconds < 50 * 60: - minutes = round(seconds / 60.0) - return _("1 minute ago", "%(minutes)d minutes ago", - minutes) % {"minutes": minutes} - - hours = round(seconds / (60.0 * 60)) - return _("1 hour ago", "%(hours)d hours ago", - hours) % {"hours": hours} - - if days == 0: - format = _("%(time)s") - elif days == 1 and local_date.day == local_yesterday.day and \ - relative: - format = _("yesterday") if shorter else \ - _("yesterday at %(time)s") - elif days < 5: - format = _("%(weekday)s") if shorter else \ - _("%(weekday)s at %(time)s") - elif days < 334: # 11mo, since confusing for same month last year - format = _("%(month_name)s %(day)s") if shorter else \ - _("%(month_name)s %(day)s at %(time)s") - - if format is None: - format = _("%(month_name)s %(day)s, %(year)s") if shorter else \ - _("%(month_name)s %(day)s, %(year)s at %(time)s") - - tfhour_clock = self.code not in ("en", "en_US", "zh_CN") - if tfhour_clock: - str_time = "%d:%02d" % (local_date.hour, local_date.minute) - elif self.code == "zh_CN": - str_time = "%s%d:%02d" % ( - (u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12], - local_date.hour % 12 or 12, local_date.minute) - else: - str_time = "%d:%02d %s" % ( - local_date.hour % 12 or 12, local_date.minute, - ("am", "pm")[local_date.hour >= 12]) - - return format % { - "month_name": self._months[local_date.month - 1], - "weekday": self._weekdays[local_date.weekday()], - "day": str(local_date.day), - "year": str(local_date.year), - "time": str_time - } - - def format_day(self, date, gmt_offset=0, dow=True): - """Formats the given date as a day of week. - - Example: "Monday, January 22". You can remove the day of week with - ``dow=False``. - """ - local_date = date - datetime.timedelta(minutes=gmt_offset) - _ = self.translate - if dow: - return _("%(weekday)s, %(month_name)s %(day)s") % { - "month_name": self._months[local_date.month - 1], - "weekday": self._weekdays[local_date.weekday()], - "day": str(local_date.day), - } - else: - return _("%(month_name)s %(day)s") % { - "month_name": self._months[local_date.month - 1], - "day": str(local_date.day), - } - - def list(self, parts): - """Returns a comma-separated list for the given list of parts. - - The format is, e.g., "A, B and C", "A and B" or just "A" for lists - of size 1. - """ - _ = self.translate - if len(parts) == 0: - return "" - if len(parts) == 1: - return parts[0] - comma = u' \u0648 ' if self.code.startswith("fa") else u", " - return _("%(commas)s and %(last)s") % { - "commas": comma.join(parts[:-1]), - "last": parts[len(parts) - 1], - } - - def friendly_number(self, value): - """Returns a comma-separated number for the given integer.""" - if self.code not in ("en", "en_US"): - return str(value) - value = str(value) - parts = [] - while value: - parts.append(value[-3:]) - value = value[:-3] - return ",".join(reversed(parts)) - - -class CSVLocale(Locale): - """Locale implementation using tornado's CSV translation format.""" - def translate(self, message, plural_message=None, count=None): - if plural_message is not None: - assert count is not None - if count != 1: - message = plural_message - message_dict = self.translations.get("plural", {}) - else: - message_dict = self.translations.get("singular", {}) - else: - message_dict = self.translations.get("unknown", {}) - return message_dict.get(message, message) - - def pgettext(self, context, message, plural_message=None, count=None): - if self.translations: - gen_log.warning('pgettext is not supported by CSVLocale') - return self.translate(message, plural_message, count) - - -class GettextLocale(Locale): - """Locale implementation using the `gettext` module.""" - def __init__(self, code, translations): - try: - # python 2 - self.ngettext = translations.ungettext - self.gettext = translations.ugettext - except AttributeError: - # python 3 - self.ngettext = translations.ngettext - self.gettext = translations.gettext - # self.gettext must exist before __init__ is called, since it - # calls into self.translate - super(GettextLocale, self).__init__(code, translations) - - def translate(self, message, plural_message=None, count=None): - if plural_message is not None: - assert count is not None - return self.ngettext(message, plural_message, count) - else: - return self.gettext(message) - - def pgettext(self, context, message, plural_message=None, count=None): - """Allows to set context for translation, accepts plural forms. - - Usage example:: - - pgettext("law", "right") - pgettext("good", "right") - - Plural message example:: - - pgettext("organization", "club", "clubs", len(clubs)) - pgettext("stick", "club", "clubs", len(clubs)) - - To generate POT file with context, add following options to step 1 - of `load_gettext_translations` sequence:: - - xgettext [basic options] --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3 - - .. versionadded:: 4.2 - """ - if plural_message is not None: - assert count is not None - msgs_with_ctxt = ("%s%s%s" % (context, CONTEXT_SEPARATOR, message), - "%s%s%s" % (context, CONTEXT_SEPARATOR, plural_message), - count) - result = self.ngettext(*msgs_with_ctxt) - if CONTEXT_SEPARATOR in result: - # Translation not found - result = self.ngettext(message, plural_message, count) - return result - else: - msg_with_ctxt = "%s%s%s" % (context, CONTEXT_SEPARATOR, message) - result = self.gettext(msg_with_ctxt) - if CONTEXT_SEPARATOR in result: - # Translation not found - result = message - return result diff --git a/lib/tornado/locks.py b/lib/tornado/locks.py deleted file mode 100755 index 9566a457..00000000 --- a/lib/tornado/locks.py +++ /dev/null @@ -1,526 +0,0 @@ -# Copyright 2015 The Tornado Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import, division, print_function - -import collections -from concurrent.futures import CancelledError - -from tornado import gen, ioloop -from tornado.concurrent import Future, future_set_result_unless_cancelled - -__all__ = ['Condition', 'Event', 'Semaphore', 'BoundedSemaphore', 'Lock'] - - -class _TimeoutGarbageCollector(object): - """Base class for objects that periodically clean up timed-out waiters. - - Avoids memory leak in a common pattern like: - - while True: - yield condition.wait(short_timeout) - print('looping....') - """ - def __init__(self): - self._waiters = collections.deque() # Futures. - self._timeouts = 0 - - def _garbage_collect(self): - # Occasionally clear timed-out waiters. - self._timeouts += 1 - if self._timeouts > 100: - self._timeouts = 0 - self._waiters = collections.deque( - w for w in self._waiters if not w.done()) - - -class Condition(_TimeoutGarbageCollector): - """A condition allows one or more coroutines to wait until notified. - - Like a standard `threading.Condition`, but does not need an underlying lock - that is acquired and released. - - With a `Condition`, coroutines can wait to be notified by other coroutines: - - .. testcode:: - - from tornado import gen - from tornado.ioloop import IOLoop - from tornado.locks import Condition - - condition = Condition() - - async def waiter(): - print("I'll wait right here") - await condition.wait() - print("I'm done waiting") - - async def notifier(): - print("About to notify") - condition.notify() - print("Done notifying") - - async def runner(): - # Wait for waiter() and notifier() in parallel - await gen.multi([waiter(), notifier()]) - - IOLoop.current().run_sync(runner) - - .. testoutput:: - - I'll wait right here - About to notify - Done notifying - I'm done waiting - - `wait` takes an optional ``timeout`` argument, which is either an absolute - timestamp:: - - io_loop = IOLoop.current() - - # Wait up to 1 second for a notification. - await condition.wait(timeout=io_loop.time() + 1) - - ...or a `datetime.timedelta` for a timeout relative to the current time:: - - # Wait up to 1 second. - await condition.wait(timeout=datetime.timedelta(seconds=1)) - - The method returns False if there's no notification before the deadline. - - .. versionchanged:: 5.0 - Previously, waiters could be notified synchronously from within - `notify`. Now, the notification will always be received on the - next iteration of the `.IOLoop`. - """ - - def __init__(self): - super(Condition, self).__init__() - self.io_loop = ioloop.IOLoop.current() - - def __repr__(self): - result = '<%s' % (self.__class__.__name__, ) - if self._waiters: - result += ' waiters[%s]' % len(self._waiters) - return result + '>' - - def wait(self, timeout=None): - """Wait for `.notify`. - - Returns a `.Future` that resolves ``True`` if the condition is notified, - or ``False`` after a timeout. - """ - waiter = Future() - self._waiters.append(waiter) - if timeout: - def on_timeout(): - if not waiter.done(): - future_set_result_unless_cancelled(waiter, False) - self._garbage_collect() - io_loop = ioloop.IOLoop.current() - timeout_handle = io_loop.add_timeout(timeout, on_timeout) - waiter.add_done_callback( - lambda _: io_loop.remove_timeout(timeout_handle)) - return waiter - - def notify(self, n=1): - """Wake ``n`` waiters.""" - waiters = [] # Waiters we plan to run right now. - while n and self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): # Might have timed out. - n -= 1 - waiters.append(waiter) - - for waiter in waiters: - future_set_result_unless_cancelled(waiter, True) - - def notify_all(self): - """Wake all waiters.""" - self.notify(len(self._waiters)) - - -class Event(object): - """An event blocks coroutines until its internal flag is set to True. - - Similar to `threading.Event`. - - A coroutine can wait for an event to be set. Once it is set, calls to - ``yield event.wait()`` will not block unless the event has been cleared: - - .. testcode:: - - from tornado import gen - from tornado.ioloop import IOLoop - from tornado.locks import Event - - event = Event() - - async def waiter(): - print("Waiting for event") - await event.wait() - print("Not waiting this time") - await event.wait() - print("Done") - - async def setter(): - print("About to set the event") - event.set() - - async def runner(): - await gen.multi([waiter(), setter()]) - - IOLoop.current().run_sync(runner) - - .. testoutput:: - - Waiting for event - About to set the event - Not waiting this time - Done - """ - def __init__(self): - self._value = False - self._waiters = set() - - def __repr__(self): - return '<%s %s>' % ( - self.__class__.__name__, 'set' if self.is_set() else 'clear') - - def is_set(self): - """Return ``True`` if the internal flag is true.""" - return self._value - - def set(self): - """Set the internal flag to ``True``. All waiters are awakened. - - Calling `.wait` once the flag is set will not block. - """ - if not self._value: - self._value = True - - for fut in self._waiters: - if not fut.done(): - fut.set_result(None) - - def clear(self): - """Reset the internal flag to ``False``. - - Calls to `.wait` will block until `.set` is called. - """ - self._value = False - - def wait(self, timeout=None): - """Block until the internal flag is true. - - Returns a Future, which raises `tornado.util.TimeoutError` after a - timeout. - """ - fut = Future() - if self._value: - fut.set_result(None) - return fut - self._waiters.add(fut) - fut.add_done_callback(lambda fut: self._waiters.remove(fut)) - if timeout is None: - return fut - else: - timeout_fut = gen.with_timeout(timeout, fut, quiet_exceptions=(CancelledError,)) - # This is a slightly clumsy workaround for the fact that - # gen.with_timeout doesn't cancel its futures. Cancelling - # fut will remove it from the waiters list. - timeout_fut.add_done_callback(lambda tf: fut.cancel() if not fut.done() else None) - return timeout_fut - - -class _ReleasingContextManager(object): - """Releases a Lock or Semaphore at the end of a "with" statement. - - with (yield semaphore.acquire()): - pass - - # Now semaphore.release() has been called. - """ - def __init__(self, obj): - self._obj = obj - - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_val, exc_tb): - self._obj.release() - - -class Semaphore(_TimeoutGarbageCollector): - """A lock that can be acquired a fixed number of times before blocking. - - A Semaphore manages a counter representing the number of `.release` calls - minus the number of `.acquire` calls, plus an initial value. The `.acquire` - method blocks if necessary until it can return without making the counter - negative. - - Semaphores limit access to a shared resource. To allow access for two - workers at a time: - - .. testsetup:: semaphore - - from collections import deque - - from tornado import gen - from tornado.ioloop import IOLoop - from tornado.concurrent import Future - - # Ensure reliable doctest output: resolve Futures one at a time. - futures_q = deque([Future() for _ in range(3)]) - - async def simulator(futures): - for f in futures: - # simulate the asynchronous passage of time - await gen.sleep(0) - await gen.sleep(0) - f.set_result(None) - - IOLoop.current().add_callback(simulator, list(futures_q)) - - def use_some_resource(): - return futures_q.popleft() - - .. testcode:: semaphore - - from tornado import gen - from tornado.ioloop import IOLoop - from tornado.locks import Semaphore - - sem = Semaphore(2) - - async def worker(worker_id): - await sem.acquire() - try: - print("Worker %d is working" % worker_id) - await use_some_resource() - finally: - print("Worker %d is done" % worker_id) - sem.release() - - async def runner(): - # Join all workers. - await gen.multi([worker(i) for i in range(3)]) - - IOLoop.current().run_sync(runner) - - .. testoutput:: semaphore - - Worker 0 is working - Worker 1 is working - Worker 0 is done - Worker 2 is working - Worker 1 is done - Worker 2 is done - - Workers 0 and 1 are allowed to run concurrently, but worker 2 waits until - the semaphore has been released once, by worker 0. - - The semaphore can be used as an async context manager:: - - async def worker(worker_id): - async with sem: - print("Worker %d is working" % worker_id) - await use_some_resource() - - # Now the semaphore has been released. - print("Worker %d is done" % worker_id) - - For compatibility with older versions of Python, `.acquire` is a - context manager, so ``worker`` could also be written as:: - - @gen.coroutine - def worker(worker_id): - with (yield sem.acquire()): - print("Worker %d is working" % worker_id) - yield use_some_resource() - - # Now the semaphore has been released. - print("Worker %d is done" % worker_id) - - .. versionchanged:: 4.3 - Added ``async with`` support in Python 3.5. - - """ - def __init__(self, value=1): - super(Semaphore, self).__init__() - if value < 0: - raise ValueError('semaphore initial value must be >= 0') - - self._value = value - - def __repr__(self): - res = super(Semaphore, self).__repr__() - extra = 'locked' if self._value == 0 else 'unlocked,value:{0}'.format( - self._value) - if self._waiters: - extra = '{0},waiters:{1}'.format(extra, len(self._waiters)) - return '<{0} [{1}]>'.format(res[1:-1], extra) - - def release(self): - """Increment the counter and wake one waiter.""" - self._value += 1 - while self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): - self._value -= 1 - - # If the waiter is a coroutine paused at - # - # with (yield semaphore.acquire()): - # - # then the context manager's __exit__ calls release() at the end - # of the "with" block. - waiter.set_result(_ReleasingContextManager(self)) - break - - def acquire(self, timeout=None): - """Decrement the counter. Returns a Future. - - Block if the counter is zero and wait for a `.release`. The Future - raises `.TimeoutError` after the deadline. - """ - waiter = Future() - if self._value > 0: - self._value -= 1 - waiter.set_result(_ReleasingContextManager(self)) - else: - self._waiters.append(waiter) - if timeout: - def on_timeout(): - if not waiter.done(): - waiter.set_exception(gen.TimeoutError()) - self._garbage_collect() - io_loop = ioloop.IOLoop.current() - timeout_handle = io_loop.add_timeout(timeout, on_timeout) - waiter.add_done_callback( - lambda _: io_loop.remove_timeout(timeout_handle)) - return waiter - - def __enter__(self): - raise RuntimeError( - "Use Semaphore like 'with (yield semaphore.acquire())', not like" - " 'with semaphore'") - - __exit__ = __enter__ - - @gen.coroutine - def __aenter__(self): - yield self.acquire() - - @gen.coroutine - def __aexit__(self, typ, value, tb): - self.release() - - -class BoundedSemaphore(Semaphore): - """A semaphore that prevents release() being called too many times. - - If `.release` would increment the semaphore's value past the initial - value, it raises `ValueError`. Semaphores are mostly used to guard - resources with limited capacity, so a semaphore released too many times - is a sign of a bug. - """ - def __init__(self, value=1): - super(BoundedSemaphore, self).__init__(value=value) - self._initial_value = value - - def release(self): - """Increment the counter and wake one waiter.""" - if self._value >= self._initial_value: - raise ValueError("Semaphore released too many times") - super(BoundedSemaphore, self).release() - - -class Lock(object): - """A lock for coroutines. - - A Lock begins unlocked, and `acquire` locks it immediately. While it is - locked, a coroutine that yields `acquire` waits until another coroutine - calls `release`. - - Releasing an unlocked lock raises `RuntimeError`. - - A Lock can be used as an async context manager with the ``async - with`` statement: - - >>> from tornado import locks - >>> lock = locks.Lock() - >>> - >>> async def f(): - ... async with lock: - ... # Do something holding the lock. - ... pass - ... - ... # Now the lock is released. - - For compatibility with older versions of Python, the `.acquire` - method asynchronously returns a regular context manager: - - >>> async def f2(): - ... with (yield lock.acquire()): - ... # Do something holding the lock. - ... pass - ... - ... # Now the lock is released. - - .. versionchanged:: 4.3 - Added ``async with`` support in Python 3.5. - - """ - def __init__(self): - self._block = BoundedSemaphore(value=1) - - def __repr__(self): - return "<%s _block=%s>" % ( - self.__class__.__name__, - self._block) - - def acquire(self, timeout=None): - """Attempt to lock. Returns a Future. - - Returns a Future, which raises `tornado.util.TimeoutError` after a - timeout. - """ - return self._block.acquire(timeout) - - def release(self): - """Unlock. - - The first coroutine in line waiting for `acquire` gets the lock. - - If not locked, raise a `RuntimeError`. - """ - try: - self._block.release() - except ValueError: - raise RuntimeError('release unlocked lock') - - def __enter__(self): - raise RuntimeError( - "Use Lock like 'with (yield lock)', not like 'with lock'") - - __exit__ = __enter__ - - @gen.coroutine - def __aenter__(self): - yield self.acquire() - - @gen.coroutine - def __aexit__(self, typ, value, tb): - self.release() diff --git a/lib/tornado/log.py b/lib/tornado/log.py deleted file mode 100755 index cda905c9..00000000 --- a/lib/tornado/log.py +++ /dev/null @@ -1,290 +0,0 @@ -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Logging support for Tornado. - -Tornado uses three logger streams: - -* ``tornado.access``: Per-request logging for Tornado's HTTP servers (and - potentially other servers in the future) -* ``tornado.application``: Logging of errors from application code (i.e. - uncaught exceptions from callbacks) -* ``tornado.general``: General-purpose logging, including any errors - or warnings from Tornado itself. - -These streams may be configured independently using the standard library's -`logging` module. For example, you may wish to send ``tornado.access`` logs -to a separate file for analysis. -""" -from __future__ import absolute_import, division, print_function - -import logging -import logging.handlers -import sys - -from tornado.escape import _unicode -from tornado.util import unicode_type, basestring_type - -try: - import colorama -except ImportError: - colorama = None - -try: - import curses # type: ignore -except ImportError: - curses = None - -# Logger objects for internal tornado use -access_log = logging.getLogger("tornado.access") -app_log = logging.getLogger("tornado.application") -gen_log = logging.getLogger("tornado.general") - - -def _stderr_supports_color(): - try: - if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): - if curses: - curses.setupterm() - if curses.tigetnum("colors") > 0: - return True - elif colorama: - if sys.stderr is getattr(colorama.initialise, 'wrapped_stderr', - object()): - return True - except Exception: - # Very broad exception handling because it's always better to - # fall back to non-colored logs than to break at startup. - pass - return False - - -def _safe_unicode(s): - try: - return _unicode(s) - except UnicodeDecodeError: - return repr(s) - - -class LogFormatter(logging.Formatter): - """Log formatter used in Tornado. - - Key features of this formatter are: - - * Color support when logging to a terminal that supports it. - * Timestamps on every log line. - * Robust against str/bytes encoding problems. - - This formatter is enabled automatically by - `tornado.options.parse_command_line` or `tornado.options.parse_config_file` - (unless ``--logging=none`` is used). - - Color support on Windows versions that do not support ANSI color codes is - enabled by use of the colorama__ library. Applications that wish to use - this must first initialize colorama with a call to ``colorama.init``. - See the colorama documentation for details. - - __ https://pypi.python.org/pypi/colorama - - .. versionchanged:: 4.5 - Added support for ``colorama``. Changed the constructor - signature to be compatible with `logging.config.dictConfig`. - """ - DEFAULT_FORMAT = \ - '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s' - DEFAULT_DATE_FORMAT = '%y%m%d %H:%M:%S' - DEFAULT_COLORS = { - logging.DEBUG: 4, # Blue - logging.INFO: 2, # Green - logging.WARNING: 3, # Yellow - logging.ERROR: 1, # Red - } - - def __init__(self, fmt=DEFAULT_FORMAT, datefmt=DEFAULT_DATE_FORMAT, - style='%', color=True, colors=DEFAULT_COLORS): - r""" - :arg bool color: Enables color support. - :arg str fmt: Log message format. - It will be applied to the attributes dict of log records. The - text between ``%(color)s`` and ``%(end_color)s`` will be colored - depending on the level if color support is on. - :arg dict colors: color mappings from logging level to terminal color - code - :arg str datefmt: Datetime format. - Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. - - .. versionchanged:: 3.2 - - Added ``fmt`` and ``datefmt`` arguments. - """ - logging.Formatter.__init__(self, datefmt=datefmt) - self._fmt = fmt - - self._colors = {} - if color and _stderr_supports_color(): - if curses is not None: - # The curses module has some str/bytes confusion in - # python3. Until version 3.2.3, most methods return - # bytes, but only accept strings. In addition, we want to - # output these strings with the logging module, which - # works with unicode strings. The explicit calls to - # unicode() below are harmless in python2 but will do the - # right conversion in python 3. - fg_color = (curses.tigetstr("setaf") or - curses.tigetstr("setf") or "") - if (3, 0) < sys.version_info < (3, 2, 3): - fg_color = unicode_type(fg_color, "ascii") - - for levelno, code in colors.items(): - self._colors[levelno] = unicode_type(curses.tparm(fg_color, code), "ascii") - self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") - else: - # If curses is not present (currently we'll only get here for - # colorama on windows), assume hard-coded ANSI color codes. - for levelno, code in colors.items(): - self._colors[levelno] = '\033[2;3%dm' % code - self._normal = '\033[0m' - else: - self._normal = '' - - def format(self, record): - try: - message = record.getMessage() - assert isinstance(message, basestring_type) # guaranteed by logging - # Encoding notes: The logging module prefers to work with character - # strings, but only enforces that log messages are instances of - # basestring. In python 2, non-ascii bytestrings will make - # their way through the logging framework until they blow up with - # an unhelpful decoding error (with this formatter it happens - # when we attach the prefix, but there are other opportunities for - # exceptions further along in the framework). - # - # If a byte string makes it this far, convert it to unicode to - # ensure it will make it out to the logs. Use repr() as a fallback - # to ensure that all byte strings can be converted successfully, - # but don't do it by default so we don't add extra quotes to ascii - # bytestrings. This is a bit of a hacky place to do this, but - # it's worth it since the encoding errors that would otherwise - # result are so useless (and tornado is fond of using utf8-encoded - # byte strings wherever possible). - record.message = _safe_unicode(message) - except Exception as e: - record.message = "Bad message (%r): %r" % (e, record.__dict__) - - record.asctime = self.formatTime(record, self.datefmt) - - if record.levelno in self._colors: - record.color = self._colors[record.levelno] - record.end_color = self._normal - else: - record.color = record.end_color = '' - - formatted = self._fmt % record.__dict__ - - if record.exc_info: - if not record.exc_text: - record.exc_text = self.formatException(record.exc_info) - if record.exc_text: - # exc_text contains multiple lines. We need to _safe_unicode - # each line separately so that non-utf8 bytes don't cause - # all the newlines to turn into '\n'. - lines = [formatted.rstrip()] - lines.extend(_safe_unicode(ln) for ln in record.exc_text.split('\n')) - formatted = '\n'.join(lines) - return formatted.replace("\n", "\n ") - - -def enable_pretty_logging(options=None, logger=None): - """Turns on formatted logging output as configured. - - This is called automatically by `tornado.options.parse_command_line` - and `tornado.options.parse_config_file`. - """ - if options is None: - import tornado.options - options = tornado.options.options - if options.logging is None or options.logging.lower() == 'none': - return - if logger is None: - logger = logging.getLogger() - logger.setLevel(getattr(logging, options.logging.upper())) - if options.log_file_prefix: - rotate_mode = options.log_rotate_mode - if rotate_mode == 'size': - channel = logging.handlers.RotatingFileHandler( - filename=options.log_file_prefix, - maxBytes=options.log_file_max_size, - backupCount=options.log_file_num_backups) - elif rotate_mode == 'time': - channel = logging.handlers.TimedRotatingFileHandler( - filename=options.log_file_prefix, - when=options.log_rotate_when, - interval=options.log_rotate_interval, - backupCount=options.log_file_num_backups) - else: - error_message = 'The value of log_rotate_mode option should be ' +\ - '"size" or "time", not "%s".' % rotate_mode - raise ValueError(error_message) - channel.setFormatter(LogFormatter(color=False)) - logger.addHandler(channel) - - if (options.log_to_stderr or - (options.log_to_stderr is None and not logger.handlers)): - # Set up color if we are in a tty and curses is installed - channel = logging.StreamHandler() - channel.setFormatter(LogFormatter()) - logger.addHandler(channel) - - -def define_logging_options(options=None): - """Add logging-related flags to ``options``. - - These options are present automatically on the default options instance; - this method is only necessary if you have created your own `.OptionParser`. - - .. versionadded:: 4.2 - This function existed in prior versions but was broken and undocumented until 4.2. - """ - if options is None: - # late import to prevent cycle - import tornado.options - options = tornado.options.options - options.define("logging", default="info", - help=("Set the Python log level. If 'none', tornado won't touch the " - "logging configuration."), - metavar="debug|info|warning|error|none") - options.define("log_to_stderr", type=bool, default=None, - help=("Send log output to stderr (colorized if possible). " - "By default use stderr if --log_file_prefix is not set and " - "no other logging is configured.")) - options.define("log_file_prefix", type=str, default=None, metavar="PATH", - help=("Path prefix for log files. " - "Note that if you are running multiple tornado processes, " - "log_file_prefix must be different for each of them (e.g. " - "include the port number)")) - options.define("log_file_max_size", type=int, default=100 * 1000 * 1000, - help="max size of log files before rollover") - options.define("log_file_num_backups", type=int, default=10, - help="number of log files to keep") - - options.define("log_rotate_when", type=str, default='midnight', - help=("specify the type of TimedRotatingFileHandler interval " - "other options:('S', 'M', 'H', 'D', 'W0'-'W6')")) - options.define("log_rotate_interval", type=int, default=1, - help="The interval value of timed rotating") - - options.define("log_rotate_mode", type=str, default='size', - help="The mode of rotating files(time or size)") - - options.add_parse_callback(lambda: enable_pretty_logging(options)) diff --git a/lib/tornado/netutil.py b/lib/tornado/netutil.py deleted file mode 100755 index e63683ad..00000000 --- a/lib/tornado/netutil.py +++ /dev/null @@ -1,575 +0,0 @@ -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Miscellaneous network utility code.""" - -from __future__ import absolute_import, division, print_function - -import errno -import os -import sys -import socket -import stat - -from tornado.concurrent import dummy_executor, run_on_executor -from tornado import gen -from tornado.ioloop import IOLoop -from tornado.platform.auto import set_close_exec -from tornado.util import PY3, Configurable, errno_from_exception - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine - ssl = None - -if PY3: - xrange = range - -if ssl is not None: - # Note that the naming of ssl.Purpose is confusing; the purpose - # of a context is to authentiate the opposite side of the connection. - _client_ssl_defaults = ssl.create_default_context( - ssl.Purpose.SERVER_AUTH) - _server_ssl_defaults = ssl.create_default_context( - ssl.Purpose.CLIENT_AUTH) - if hasattr(ssl, 'OP_NO_COMPRESSION'): - # See netutil.ssl_options_to_context - _client_ssl_defaults.options |= ssl.OP_NO_COMPRESSION - _server_ssl_defaults.options |= ssl.OP_NO_COMPRESSION -else: - # Google App Engine - _client_ssl_defaults = dict(cert_reqs=None, - ca_certs=None) - _server_ssl_defaults = {} - -# ThreadedResolver runs getaddrinfo on a thread. If the hostname is unicode, -# getaddrinfo attempts to import encodings.idna. If this is done at -# module-import time, the import lock is already held by the main thread, -# leading to deadlock. Avoid it by caching the idna encoder on the main -# thread now. -u'foo'.encode('idna') - -# For undiagnosed reasons, 'latin1' codec may also need to be preloaded. -u'foo'.encode('latin1') - -# These errnos indicate that a non-blocking operation must be retried -# at a later time. On most platforms they're the same value, but on -# some they differ. -_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) - -if hasattr(errno, "WSAEWOULDBLOCK"): - _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) # type: ignore - -# Default backlog used when calling sock.listen() -_DEFAULT_BACKLOG = 128 - - -def bind_sockets(port, address=None, family=socket.AF_UNSPEC, - backlog=_DEFAULT_BACKLOG, flags=None, reuse_port=False): - """Creates listening sockets bound to the given port and address. - - Returns a list of socket objects (multiple sockets are returned if - the given address maps to multiple IP addresses, which is most common - for mixed IPv4 and IPv6 use). - - Address may be either an IP address or hostname. If it's a hostname, - the server will listen on all IP addresses associated with the - name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either `socket.AF_INET` - or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise - both will be used if available. - - The ``backlog`` argument has the same meaning as for - `socket.listen() <socket.socket.listen>`. - - ``flags`` is a bitmask of AI_* flags to `~socket.getaddrinfo`, like - ``socket.AI_PASSIVE | socket.AI_NUMERICHOST``. - - ``reuse_port`` option sets ``SO_REUSEPORT`` option for every socket - in the list. If your platform doesn't support this option ValueError will - be raised. - """ - if reuse_port and not hasattr(socket, "SO_REUSEPORT"): - raise ValueError("the platform doesn't support SO_REUSEPORT") - - sockets = [] - if address == "": - address = None - if not socket.has_ipv6 and family == socket.AF_UNSPEC: - # Python can be compiled with --disable-ipv6, which causes - # operations on AF_INET6 sockets to fail, but does not - # automatically exclude those results from getaddrinfo - # results. - # http://bugs.python.org/issue16208 - family = socket.AF_INET - if flags is None: - flags = socket.AI_PASSIVE - bound_port = None - for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, - 0, flags)): - af, socktype, proto, canonname, sockaddr = res - if (sys.platform == 'darwin' and address == 'localhost' and - af == socket.AF_INET6 and sockaddr[3] != 0): - # Mac OS X includes a link-local address fe80::1%lo0 in the - # getaddrinfo results for 'localhost'. However, the firewall - # doesn't understand that this is a local address and will - # prompt for access (often repeatedly, due to an apparent - # bug in its ability to remember granting access to an - # application). Skip these addresses. - continue - try: - sock = socket.socket(af, socktype, proto) - except socket.error as e: - if errno_from_exception(e) == errno.EAFNOSUPPORT: - continue - raise - set_close_exec(sock.fileno()) - if os.name != 'nt': - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except socket.error as e: - if errno_from_exception(e) != errno.ENOPROTOOPT: - # Hurd doesn't support SO_REUSEADDR. - raise - if reuse_port: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if af == socket.AF_INET6: - # On linux, ipv6 sockets accept ipv4 too by default, - # but this makes it impossible to bind to both - # 0.0.0.0 in ipv4 and :: in ipv6. On other systems, - # separate sockets *must* be used to listen for both ipv4 - # and ipv6. For consistency, always disable ipv4 on our - # ipv6 sockets and use a separate ipv4 socket when needed. - # - # Python 2.x on windows doesn't have IPPROTO_IPV6. - if hasattr(socket, "IPPROTO_IPV6"): - sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) - - # automatic port allocation with port=None - # should bind on the same port on IPv4 and IPv6 - host, requested_port = sockaddr[:2] - if requested_port == 0 and bound_port is not None: - sockaddr = tuple([host, bound_port] + list(sockaddr[2:])) - - sock.setblocking(0) - sock.bind(sockaddr) - bound_port = sock.getsockname()[1] - sock.listen(backlog) - sockets.append(sock) - return sockets - - -if hasattr(socket, 'AF_UNIX'): - def bind_unix_socket(file, mode=0o600, backlog=_DEFAULT_BACKLOG): - """Creates a listening unix socket. - - If a socket with the given name already exists, it will be deleted. - If any other file with that name exists, an exception will be - raised. - - Returns a socket object (not a list of socket objects like - `bind_sockets`) - """ - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - set_close_exec(sock.fileno()) - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except socket.error as e: - if errno_from_exception(e) != errno.ENOPROTOOPT: - # Hurd doesn't support SO_REUSEADDR - raise - sock.setblocking(0) - try: - st = os.stat(file) - except OSError as err: - if errno_from_exception(err) != errno.ENOENT: - raise - else: - if stat.S_ISSOCK(st.st_mode): - os.remove(file) - else: - raise ValueError("File %s exists and is not a socket", file) - sock.bind(file) - os.chmod(file, mode) - sock.listen(backlog) - return sock - - -def add_accept_handler(sock, callback): - """Adds an `.IOLoop` event handler to accept new connections on ``sock``. - - When a connection is accepted, ``callback(connection, address)`` will - be run (``connection`` is a socket object, and ``address`` is the - address of the other end of the connection). Note that this signature - is different from the ``callback(fd, events)`` signature used for - `.IOLoop` handlers. - - A callable is returned which, when called, will remove the `.IOLoop` - event handler and stop processing further incoming connections. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - .. versionchanged:: 5.0 - A callable is returned (``None`` was returned before). - """ - io_loop = IOLoop.current() - removed = [False] - - def accept_handler(fd, events): - # More connections may come in while we're handling callbacks; - # to prevent starvation of other tasks we must limit the number - # of connections we accept at a time. Ideally we would accept - # up to the number of connections that were waiting when we - # entered this method, but this information is not available - # (and rearranging this method to call accept() as many times - # as possible before running any callbacks would have adverse - # effects on load balancing in multiprocess configurations). - # Instead, we use the (default) listen backlog as a rough - # heuristic for the number of connections we can reasonably - # accept at once. - for i in xrange(_DEFAULT_BACKLOG): - if removed[0]: - # The socket was probably closed - return - try: - connection, address = sock.accept() - except socket.error as e: - # _ERRNO_WOULDBLOCK indicate we have accepted every - # connection that is available. - if errno_from_exception(e) in _ERRNO_WOULDBLOCK: - return - # ECONNABORTED indicates that there was a connection - # but it was closed while still in the accept queue. - # (observed on FreeBSD). - if errno_from_exception(e) == errno.ECONNABORTED: - continue - raise - set_close_exec(connection.fileno()) - callback(connection, address) - - def remove_handler(): - io_loop.remove_handler(sock) - removed[0] = True - - io_loop.add_handler(sock, accept_handler, IOLoop.READ) - return remove_handler - - -def is_valid_ip(ip): - """Returns true if the given string is a well-formed IP address. - - Supports IPv4 and IPv6. - """ - if not ip or '\x00' in ip: - # getaddrinfo resolves empty strings to localhost, and truncates - # on zero bytes. - return False - try: - res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC, - socket.SOCK_STREAM, - 0, socket.AI_NUMERICHOST) - return bool(res) - except socket.gaierror as e: - if e.args[0] == socket.EAI_NONAME: - return False - raise - return True - - -class Resolver(Configurable): - """Configurable asynchronous DNS resolver interface. - - By default, a blocking implementation is used (which simply calls - `socket.getaddrinfo`). An alternative implementation can be - chosen with the `Resolver.configure <.Configurable.configure>` - class method:: - - Resolver.configure('tornado.netutil.ThreadedResolver') - - The implementations of this interface included with Tornado are - - * `tornado.netutil.DefaultExecutorResolver` - * `tornado.netutil.BlockingResolver` (deprecated) - * `tornado.netutil.ThreadedResolver` (deprecated) - * `tornado.netutil.OverrideResolver` - * `tornado.platform.twisted.TwistedResolver` - * `tornado.platform.caresresolver.CaresResolver` - - .. versionchanged:: 5.0 - The default implementation has changed from `BlockingResolver` to - `DefaultExecutorResolver`. - """ - @classmethod - def configurable_base(cls): - return Resolver - - @classmethod - def configurable_default(cls): - return DefaultExecutorResolver - - def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None): - """Resolves an address. - - The ``host`` argument is a string which may be a hostname or a - literal IP address. - - Returns a `.Future` whose result is a list of (family, - address) pairs, where address is a tuple suitable to pass to - `socket.connect <socket.socket.connect>` (i.e. a ``(host, - port)`` pair for IPv4; additional fields may be present for - IPv6). If a ``callback`` is passed, it will be run with the - result as an argument when it is complete. - - :raises IOError: if the address cannot be resolved. - - .. versionchanged:: 4.4 - Standardized all implementations to raise `IOError`. - - .. deprecated:: 5.1 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. - """ - raise NotImplementedError() - - def close(self): - """Closes the `Resolver`, freeing any resources used. - - .. versionadded:: 3.1 - - """ - pass - - -def _resolve_addr(host, port, family=socket.AF_UNSPEC): - # On Solaris, getaddrinfo fails if the given port is not found - # in /etc/services and no socket type is given, so we must pass - # one here. The socket type used here doesn't seem to actually - # matter (we discard the one we get back in the results), - # so the addresses we return should still be usable with SOCK_DGRAM. - addrinfo = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) - results = [] - for family, socktype, proto, canonname, address in addrinfo: - results.append((family, address)) - return results - - -class DefaultExecutorResolver(Resolver): - """Resolver implementation using `.IOLoop.run_in_executor`. - - .. versionadded:: 5.0 - """ - @gen.coroutine - def resolve(self, host, port, family=socket.AF_UNSPEC): - result = yield IOLoop.current().run_in_executor( - None, _resolve_addr, host, port, family) - raise gen.Return(result) - - -class ExecutorResolver(Resolver): - """Resolver implementation using a `concurrent.futures.Executor`. - - Use this instead of `ThreadedResolver` when you require additional - control over the executor being used. - - The executor will be shut down when the resolver is closed unless - ``close_resolver=False``; use this if you want to reuse the same - executor elsewhere. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - .. deprecated:: 5.0 - The default `Resolver` now uses `.IOLoop.run_in_executor`; use that instead - of this class. - """ - def initialize(self, executor=None, close_executor=True): - self.io_loop = IOLoop.current() - if executor is not None: - self.executor = executor - self.close_executor = close_executor - else: - self.executor = dummy_executor - self.close_executor = False - - def close(self): - if self.close_executor: - self.executor.shutdown() - self.executor = None - - @run_on_executor - def resolve(self, host, port, family=socket.AF_UNSPEC): - return _resolve_addr(host, port, family) - - -class BlockingResolver(ExecutorResolver): - """Default `Resolver` implementation, using `socket.getaddrinfo`. - - The `.IOLoop` will be blocked during the resolution, although the - callback will not be run until the next `.IOLoop` iteration. - - .. deprecated:: 5.0 - The default `Resolver` now uses `.IOLoop.run_in_executor`; use that instead - of this class. - """ - def initialize(self): - super(BlockingResolver, self).initialize() - - -class ThreadedResolver(ExecutorResolver): - """Multithreaded non-blocking `Resolver` implementation. - - Requires the `concurrent.futures` package to be installed - (available in the standard library since Python 3.2, - installable with ``pip install futures`` in older versions). - - The thread pool size can be configured with:: - - Resolver.configure('tornado.netutil.ThreadedResolver', - num_threads=10) - - .. versionchanged:: 3.1 - All ``ThreadedResolvers`` share a single thread pool, whose - size is set by the first one to be created. - - .. deprecated:: 5.0 - The default `Resolver` now uses `.IOLoop.run_in_executor`; use that instead - of this class. - """ - _threadpool = None # type: ignore - _threadpool_pid = None # type: int - - def initialize(self, num_threads=10): - threadpool = ThreadedResolver._create_threadpool(num_threads) - super(ThreadedResolver, self).initialize( - executor=threadpool, close_executor=False) - - @classmethod - def _create_threadpool(cls, num_threads): - pid = os.getpid() - if cls._threadpool_pid != pid: - # Threads cannot survive after a fork, so if our pid isn't what it - # was when we created the pool then delete it. - cls._threadpool = None - if cls._threadpool is None: - from concurrent.futures import ThreadPoolExecutor - cls._threadpool = ThreadPoolExecutor(num_threads) - cls._threadpool_pid = pid - return cls._threadpool - - -class OverrideResolver(Resolver): - """Wraps a resolver with a mapping of overrides. - - This can be used to make local DNS changes (e.g. for testing) - without modifying system-wide settings. - - The mapping can be in three formats:: - - { - # Hostname to host or ip - "example.com": "127.0.1.1", - - # Host+port to host+port - ("login.example.com", 443): ("localhost", 1443), - - # Host+port+address family to host+port - ("login.example.com", 443, socket.AF_INET6): ("::1", 1443), - } - - .. versionchanged:: 5.0 - Added support for host-port-family triplets. - """ - def initialize(self, resolver, mapping): - self.resolver = resolver - self.mapping = mapping - - def close(self): - self.resolver.close() - - def resolve(self, host, port, family=socket.AF_UNSPEC, *args, **kwargs): - if (host, port, family) in self.mapping: - host, port = self.mapping[(host, port, family)] - elif (host, port) in self.mapping: - host, port = self.mapping[(host, port)] - elif host in self.mapping: - host = self.mapping[host] - return self.resolver.resolve(host, port, family, *args, **kwargs) - - -# These are the keyword arguments to ssl.wrap_socket that must be translated -# to their SSLContext equivalents (the other arguments are still passed -# to SSLContext.wrap_socket). -_SSL_CONTEXT_KEYWORDS = frozenset(['ssl_version', 'certfile', 'keyfile', - 'cert_reqs', 'ca_certs', 'ciphers']) - - -def ssl_options_to_context(ssl_options): - """Try to convert an ``ssl_options`` dictionary to an - `~ssl.SSLContext` object. - - The ``ssl_options`` dictionary contains keywords to be passed to - `ssl.wrap_socket`. In Python 2.7.9+, `ssl.SSLContext` objects can - be used instead. This function converts the dict form to its - `~ssl.SSLContext` equivalent, and may be used when a component which - accepts both forms needs to upgrade to the `~ssl.SSLContext` version - to use features like SNI or NPN. - """ - if isinstance(ssl_options, ssl.SSLContext): - return ssl_options - assert isinstance(ssl_options, dict) - assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options - # Can't use create_default_context since this interface doesn't - # tell us client vs server. - context = ssl.SSLContext( - ssl_options.get('ssl_version', ssl.PROTOCOL_SSLv23)) - if 'certfile' in ssl_options: - context.load_cert_chain(ssl_options['certfile'], ssl_options.get('keyfile', None)) - if 'cert_reqs' in ssl_options: - context.verify_mode = ssl_options['cert_reqs'] - if 'ca_certs' in ssl_options: - context.load_verify_locations(ssl_options['ca_certs']) - if 'ciphers' in ssl_options: - context.set_ciphers(ssl_options['ciphers']) - if hasattr(ssl, 'OP_NO_COMPRESSION'): - # Disable TLS compression to avoid CRIME and related attacks. - # This constant depends on openssl version 1.0. - # TODO: Do we need to do this ourselves or can we trust - # the defaults? - context.options |= ssl.OP_NO_COMPRESSION - return context - - -def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): - """Returns an ``ssl.SSLSocket`` wrapping the given socket. - - ``ssl_options`` may be either an `ssl.SSLContext` object or a - dictionary (as accepted by `ssl_options_to_context`). Additional - keyword arguments are passed to ``wrap_socket`` (either the - `~ssl.SSLContext` method or the `ssl` module function as - appropriate). - """ - context = ssl_options_to_context(ssl_options) - if ssl.HAS_SNI: - # In python 3.4, wrap_socket only accepts the server_hostname - # argument if HAS_SNI is true. - # TODO: add a unittest (python added server-side SNI support in 3.4) - # In the meantime it can be manually tested with - # python3 -m tornado.httpclient https://sni.velox.ch - return context.wrap_socket(socket, server_hostname=server_hostname, - **kwargs) - else: - return context.wrap_socket(socket, **kwargs) diff --git a/lib/tornado/options.py b/lib/tornado/options.py deleted file mode 100755 index 0a4b965f..00000000 --- a/lib/tornado/options.py +++ /dev/null @@ -1,654 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A command line parsing module that lets modules define their own options. - -This module is inspired by Google's `gflags -<https://github.com/google/python-gflags>`_. The primary difference -with libraries such as `argparse` is that a global registry is used so -that options may be defined in any module (it also enables -`tornado.log` by default). The rest of Tornado does not depend on this -module, so feel free to use `argparse` or other configuration -libraries if you prefer them. - -Options must be defined with `tornado.options.define` before use, -generally at the top level of a module. The options are then -accessible as attributes of `tornado.options.options`:: - - # myapp/db.py - from tornado.options import define, options - - define("mysql_host", default="127.0.0.1:3306", help="Main user DB") - define("memcache_hosts", default="127.0.0.1:11011", multiple=True, - help="Main user memcache servers") - - def connect(): - db = database.Connection(options.mysql_host) - ... - - # myapp/server.py - from tornado.options import define, options - - define("port", default=8080, help="port to listen on") - - def start_server(): - app = make_app() - app.listen(options.port) - -The ``main()`` method of your application does not need to be aware of all of -the options used throughout your program; they are all automatically loaded -when the modules are loaded. However, all modules that define options -must have been imported before the command line is parsed. - -Your ``main()`` method can parse the command line or parse a config file with -either `parse_command_line` or `parse_config_file`:: - - import myapp.db, myapp.server - import tornado.options - - if __name__ == '__main__': - tornado.options.parse_command_line() - # or - tornado.options.parse_config_file("/etc/server.conf") - -.. note:: - - When using multiple ``parse_*`` functions, pass ``final=False`` to all - but the last one, or side effects may occur twice (in particular, - this can result in log messages being doubled). - -`tornado.options.options` is a singleton instance of `OptionParser`, and -the top-level functions in this module (`define`, `parse_command_line`, etc) -simply call methods on it. You may create additional `OptionParser` -instances to define isolated sets of options, such as for subcommands. - -.. note:: - - By default, several options are defined that will configure the - standard `logging` module when `parse_command_line` or `parse_config_file` - are called. If you want Tornado to leave the logging configuration - alone so you can manage it yourself, either pass ``--logging=none`` - on the command line or do the following to disable it in code:: - - from tornado.options import options, parse_command_line - options.logging = None - parse_command_line() - -.. versionchanged:: 4.3 - Dashes and underscores are fully interchangeable in option names; - options can be defined, set, and read with any mix of the two. - Dashes are typical for command-line usage while config files require - underscores. -""" - -from __future__ import absolute_import, division, print_function - -import datetime -import numbers -import re -import sys -import os -import textwrap - -from tornado.escape import _unicode, native_str -from tornado.log import define_logging_options -from tornado import stack_context -from tornado.util import basestring_type, exec_in - - -class Error(Exception): - """Exception raised by errors in the options module.""" - pass - - -class OptionParser(object): - """A collection of options, a dictionary with object-like access. - - Normally accessed via static functions in the `tornado.options` module, - which reference a global instance. - """ - def __init__(self): - # we have to use self.__dict__ because we override setattr. - self.__dict__['_options'] = {} - self.__dict__['_parse_callbacks'] = [] - self.define("help", type=bool, help="show this help information", - callback=self._help_callback) - - def _normalize_name(self, name): - return name.replace('_', '-') - - def __getattr__(self, name): - name = self._normalize_name(name) - if isinstance(self._options.get(name), _Option): - return self._options[name].value() - raise AttributeError("Unrecognized option %r" % name) - - def __setattr__(self, name, value): - name = self._normalize_name(name) - if isinstance(self._options.get(name), _Option): - return self._options[name].set(value) - raise AttributeError("Unrecognized option %r" % name) - - def __iter__(self): - return (opt.name for opt in self._options.values()) - - def __contains__(self, name): - name = self._normalize_name(name) - return name in self._options - - def __getitem__(self, name): - return self.__getattr__(name) - - def __setitem__(self, name, value): - return self.__setattr__(name, value) - - def items(self): - """A sequence of (name, value) pairs. - - .. versionadded:: 3.1 - """ - return [(opt.name, opt.value()) for name, opt in self._options.items()] - - def groups(self): - """The set of option-groups created by ``define``. - - .. versionadded:: 3.1 - """ - return set(opt.group_name for opt in self._options.values()) - - def group_dict(self, group): - """The names and values of options in a group. - - Useful for copying options into Application settings:: - - from tornado.options import define, parse_command_line, options - - define('template_path', group='application') - define('static_path', group='application') - - parse_command_line() - - application = Application( - handlers, **options.group_dict('application')) - - .. versionadded:: 3.1 - """ - return dict( - (opt.name, opt.value()) for name, opt in self._options.items() - if not group or group == opt.group_name) - - def as_dict(self): - """The names and values of all options. - - .. versionadded:: 3.1 - """ - return dict( - (opt.name, opt.value()) for name, opt in self._options.items()) - - def define(self, name, default=None, type=None, help=None, metavar=None, - multiple=False, group=None, callback=None): - """Defines a new command line option. - - ``type`` can be any of `str`, `int`, `float`, `bool`, - `~datetime.datetime`, or `~datetime.timedelta`. If no ``type`` - is given but a ``default`` is, ``type`` is the type of - ``default``. Otherwise, ``type`` defaults to `str`. - - If ``multiple`` is True, the option value is a list of ``type`` - instead of an instance of ``type``. - - ``help`` and ``metavar`` are used to construct the - automatically generated command line help string. The help - message is formatted like:: - - --name=METAVAR help string - - ``group`` is used to group the defined options in logical - groups. By default, command line options are grouped by the - file in which they are defined. - - Command line option names must be unique globally. - - If a ``callback`` is given, it will be run with the new value whenever - the option is changed. This can be used to combine command-line - and file-based options:: - - define("config", type=str, help="path to config file", - callback=lambda path: parse_config_file(path, final=False)) - - With this definition, options in the file specified by ``--config`` will - override options set earlier on the command line, but can be overridden - by later flags. - - """ - normalized = self._normalize_name(name) - if normalized in self._options: - raise Error("Option %r already defined in %s" % - (normalized, self._options[normalized].file_name)) - frame = sys._getframe(0) - options_file = frame.f_code.co_filename - - # Can be called directly, or through top level define() fn, in which - # case, step up above that frame to look for real caller. - if (frame.f_back.f_code.co_filename == options_file and - frame.f_back.f_code.co_name == 'define'): - frame = frame.f_back - - file_name = frame.f_back.f_code.co_filename - if file_name == options_file: - file_name = "" - if type is None: - if not multiple and default is not None: - type = default.__class__ - else: - type = str - if group: - group_name = group - else: - group_name = file_name - option = _Option(name, file_name=file_name, - default=default, type=type, help=help, - metavar=metavar, multiple=multiple, - group_name=group_name, - callback=callback) - self._options[normalized] = option - - def parse_command_line(self, args=None, final=True): - """Parses all options given on the command line (defaults to - `sys.argv`). - - Options look like ``--option=value`` and are parsed according - to their ``type``. For boolean options, ``--option`` is - equivalent to ``--option=true`` - - If the option has ``multiple=True``, comma-separated values - are accepted. For multi-value integer options, the syntax - ``x:y`` is also accepted and equivalent to ``range(x, y)``. - - Note that ``args[0]`` is ignored since it is the program name - in `sys.argv`. - - We return a list of all arguments that are not parsed as options. - - If ``final`` is ``False``, parse callbacks will not be run. - This is useful for applications that wish to combine configurations - from multiple sources. - - """ - if args is None: - args = sys.argv - remaining = [] - for i in range(1, len(args)): - # All things after the last option are command line arguments - if not args[i].startswith("-"): - remaining = args[i:] - break - if args[i] == "--": - remaining = args[i + 1:] - break - arg = args[i].lstrip("-") - name, equals, value = arg.partition("=") - name = self._normalize_name(name) - if name not in self._options: - self.print_help() - raise Error('Unrecognized command line option: %r' % name) - option = self._options[name] - if not equals: - if option.type == bool: - value = "true" - else: - raise Error('Option %r requires a value' % name) - option.parse(value) - - if final: - self.run_parse_callbacks() - - return remaining - - def parse_config_file(self, path, final=True): - """Parses and loads the config file at the given path. - - The config file contains Python code that will be executed (so - it is **not safe** to use untrusted config files). Anything in - the global namespace that matches a defined option will be - used to set that option's value. - - Options may either be the specified type for the option or - strings (in which case they will be parsed the same way as in - `.parse_command_line`) - - Example (using the options defined in the top-level docs of - this module):: - - port = 80 - mysql_host = 'mydb.example.com:3306' - # Both lists and comma-separated strings are allowed for - # multiple=True. - memcache_hosts = ['cache1.example.com:11011', - 'cache2.example.com:11011'] - memcache_hosts = 'cache1.example.com:11011,cache2.example.com:11011' - - If ``final`` is ``False``, parse callbacks will not be run. - This is useful for applications that wish to combine configurations - from multiple sources. - - .. note:: - - `tornado.options` is primarily a command-line library. - Config file support is provided for applications that wish - to use it, but applications that prefer config files may - wish to look at other libraries instead. - - .. versionchanged:: 4.1 - Config files are now always interpreted as utf-8 instead of - the system default encoding. - - .. versionchanged:: 4.4 - The special variable ``__file__`` is available inside config - files, specifying the absolute path to the config file itself. - - .. versionchanged:: 5.1 - Added the ability to set options via strings in config files. - - """ - config = {'__file__': os.path.abspath(path)} - with open(path, 'rb') as f: - exec_in(native_str(f.read()), config, config) - for name in config: - normalized = self._normalize_name(name) - if normalized in self._options: - option = self._options[normalized] - if option.multiple: - if not isinstance(config[name], (list, str)): - raise Error("Option %r is required to be a list of %s " - "or a comma-separated string" % - (option.name, option.type.__name__)) - - if type(config[name]) == str and option.type != str: - option.parse(config[name]) - else: - option.set(config[name]) - - if final: - self.run_parse_callbacks() - - def print_help(self, file=None): - """Prints all the command line options to stderr (or another file).""" - if file is None: - file = sys.stderr - print("Usage: %s [OPTIONS]" % sys.argv[0], file=file) - print("\nOptions:\n", file=file) - by_group = {} - for option in self._options.values(): - by_group.setdefault(option.group_name, []).append(option) - - for filename, o in sorted(by_group.items()): - if filename: - print("\n%s options:\n" % os.path.normpath(filename), file=file) - o.sort(key=lambda option: option.name) - for option in o: - # Always print names with dashes in a CLI context. - prefix = self._normalize_name(option.name) - if option.metavar: - prefix += "=" + option.metavar - description = option.help or "" - if option.default is not None and option.default != '': - description += " (default %s)" % option.default - lines = textwrap.wrap(description, 79 - 35) - if len(prefix) > 30 or len(lines) == 0: - lines.insert(0, '') - print(" --%-30s %s" % (prefix, lines[0]), file=file) - for line in lines[1:]: - print("%-34s %s" % (' ', line), file=file) - print(file=file) - - def _help_callback(self, value): - if value: - self.print_help() - sys.exit(0) - - def add_parse_callback(self, callback): - """Adds a parse callback, to be invoked when option parsing is done.""" - self._parse_callbacks.append(stack_context.wrap(callback)) - - def run_parse_callbacks(self): - for callback in self._parse_callbacks: - callback() - - def mockable(self): - """Returns a wrapper around self that is compatible with - `mock.patch <unittest.mock.patch>`. - - The `mock.patch <unittest.mock.patch>` function (included in - the standard library `unittest.mock` package since Python 3.3, - or in the third-party ``mock`` package for older versions of - Python) is incompatible with objects like ``options`` that - override ``__getattr__`` and ``__setattr__``. This function - returns an object that can be used with `mock.patch.object - <unittest.mock.patch.object>` to modify option values:: - - with mock.patch.object(options.mockable(), 'name', value): - assert options.name == value - """ - return _Mockable(self) - - -class _Mockable(object): - """`mock.patch` compatible wrapper for `OptionParser`. - - As of ``mock`` version 1.0.1, when an object uses ``__getattr__`` - hooks instead of ``__dict__``, ``patch.__exit__`` tries to delete - the attribute it set instead of setting a new one (assuming that - the object does not catpure ``__setattr__``, so the patch - created a new attribute in ``__dict__``). - - _Mockable's getattr and setattr pass through to the underlying - OptionParser, and delattr undoes the effect of a previous setattr. - """ - def __init__(self, options): - # Modify __dict__ directly to bypass __setattr__ - self.__dict__['_options'] = options - self.__dict__['_originals'] = {} - - def __getattr__(self, name): - return getattr(self._options, name) - - def __setattr__(self, name, value): - assert name not in self._originals, "don't reuse mockable objects" - self._originals[name] = getattr(self._options, name) - setattr(self._options, name, value) - - def __delattr__(self, name): - setattr(self._options, name, self._originals.pop(name)) - - -class _Option(object): - UNSET = object() - - def __init__(self, name, default=None, type=basestring_type, help=None, - metavar=None, multiple=False, file_name=None, group_name=None, - callback=None): - if default is None and multiple: - default = [] - self.name = name - self.type = type - self.help = help - self.metavar = metavar - self.multiple = multiple - self.file_name = file_name - self.group_name = group_name - self.callback = callback - self.default = default - self._value = _Option.UNSET - - def value(self): - return self.default if self._value is _Option.UNSET else self._value - - def parse(self, value): - _parse = { - datetime.datetime: self._parse_datetime, - datetime.timedelta: self._parse_timedelta, - bool: self._parse_bool, - basestring_type: self._parse_string, - }.get(self.type, self.type) - if self.multiple: - self._value = [] - for part in value.split(","): - if issubclass(self.type, numbers.Integral): - # allow ranges of the form X:Y (inclusive at both ends) - lo, _, hi = part.partition(":") - lo = _parse(lo) - hi = _parse(hi) if hi else lo - self._value.extend(range(lo, hi + 1)) - else: - self._value.append(_parse(part)) - else: - self._value = _parse(value) - if self.callback is not None: - self.callback(self._value) - return self.value() - - def set(self, value): - if self.multiple: - if not isinstance(value, list): - raise Error("Option %r is required to be a list of %s" % - (self.name, self.type.__name__)) - for item in value: - if item is not None and not isinstance(item, self.type): - raise Error("Option %r is required to be a list of %s" % - (self.name, self.type.__name__)) - else: - if value is not None and not isinstance(value, self.type): - raise Error("Option %r is required to be a %s (%s given)" % - (self.name, self.type.__name__, type(value))) - self._value = value - if self.callback is not None: - self.callback(self._value) - - # Supported date/time formats in our options - _DATETIME_FORMATS = [ - "%a %b %d %H:%M:%S %Y", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%d %H:%M", - "%Y-%m-%dT%H:%M", - "%Y%m%d %H:%M:%S", - "%Y%m%d %H:%M", - "%Y-%m-%d", - "%Y%m%d", - "%H:%M:%S", - "%H:%M", - ] - - def _parse_datetime(self, value): - for format in self._DATETIME_FORMATS: - try: - return datetime.datetime.strptime(value, format) - except ValueError: - pass - raise Error('Unrecognized date/time format: %r' % value) - - _TIMEDELTA_ABBREV_DICT = { - 'h': 'hours', - 'm': 'minutes', - 'min': 'minutes', - 's': 'seconds', - 'sec': 'seconds', - 'ms': 'milliseconds', - 'us': 'microseconds', - 'd': 'days', - 'w': 'weeks', - } - - _FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?' - - _TIMEDELTA_PATTERN = re.compile( - r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE) - - def _parse_timedelta(self, value): - try: - sum = datetime.timedelta() - start = 0 - while start < len(value): - m = self._TIMEDELTA_PATTERN.match(value, start) - if not m: - raise Exception() - num = float(m.group(1)) - units = m.group(2) or 'seconds' - units = self._TIMEDELTA_ABBREV_DICT.get(units, units) - sum += datetime.timedelta(**{units: num}) - start = m.end() - return sum - except Exception: - raise - - def _parse_bool(self, value): - return value.lower() not in ("false", "0", "f") - - def _parse_string(self, value): - return _unicode(value) - - -options = OptionParser() -"""Global options object. - -All defined options are available as attributes on this object. -""" - - -def define(name, default=None, type=None, help=None, metavar=None, - multiple=False, group=None, callback=None): - """Defines an option in the global namespace. - - See `OptionParser.define`. - """ - return options.define(name, default=default, type=type, help=help, - metavar=metavar, multiple=multiple, group=group, - callback=callback) - - -def parse_command_line(args=None, final=True): - """Parses global options from the command line. - - See `OptionParser.parse_command_line`. - """ - return options.parse_command_line(args, final=final) - - -def parse_config_file(path, final=True): - """Parses global options from a config file. - - See `OptionParser.parse_config_file`. - """ - return options.parse_config_file(path, final=final) - - -def print_help(file=None): - """Prints all the command line options to stderr (or another file). - - See `OptionParser.print_help`. - """ - return options.print_help(file) - - -def add_parse_callback(callback): - """Adds a parse callback, to be invoked when option parsing is done. - - See `OptionParser.add_parse_callback` - """ - options.add_parse_callback(callback) - - -# Default options -define_logging_options(options) diff --git a/lib/tornado/platform/asyncio.py b/lib/tornado/platform/asyncio.py deleted file mode 100755 index e0042e1d..00000000 --- a/lib/tornado/platform/asyncio.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Bridges between the `asyncio` module and Tornado IOLoop. - -.. versionadded:: 3.2 - -This module integrates Tornado with the ``asyncio`` module introduced -in Python 3.4. This makes it possible to combine the two libraries on -the same event loop. - -.. deprecated:: 5.0 - - While the code in this module is still used, it is now enabled - automatically when `asyncio` is available, so applications should - no longer need to refer to this module directly. - -.. note:: - - Tornado requires the `~asyncio.AbstractEventLoop.add_reader` family of - methods, so it is not compatible with the `~asyncio.ProactorEventLoop` on - Windows. Use the `~asyncio.SelectorEventLoop` instead. -""" - -from __future__ import absolute_import, division, print_function -import functools - -from tornado.gen import convert_yielded -from tornado.ioloop import IOLoop -from tornado import stack_context - -import asyncio - - -class BaseAsyncIOLoop(IOLoop): - def initialize(self, asyncio_loop, **kwargs): - self.asyncio_loop = asyncio_loop - # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler) - self.handlers = {} - # Set of fds listening for reads/writes - self.readers = set() - self.writers = set() - self.closing = False - # If an asyncio loop was closed through an asyncio interface - # instead of IOLoop.close(), we'd never hear about it and may - # have left a dangling reference in our map. In case an - # application (or, more likely, a test suite) creates and - # destroys a lot of event loops in this way, check here to - # ensure that we don't have a lot of dead loops building up in - # the map. - # - # TODO(bdarnell): consider making self.asyncio_loop a weakref - # for AsyncIOMainLoop and make _ioloop_for_asyncio a - # WeakKeyDictionary. - for loop in list(IOLoop._ioloop_for_asyncio): - if loop.is_closed(): - del IOLoop._ioloop_for_asyncio[loop] - IOLoop._ioloop_for_asyncio[asyncio_loop] = self - super(BaseAsyncIOLoop, self).initialize(**kwargs) - - def close(self, all_fds=False): - self.closing = True - for fd in list(self.handlers): - fileobj, handler_func = self.handlers[fd] - self.remove_handler(fd) - if all_fds: - self.close_fd(fileobj) - # Remove the mapping before closing the asyncio loop. If this - # happened in the other order, we could race against another - # initialize() call which would see the closed asyncio loop, - # assume it was closed from the asyncio side, and do this - # cleanup for us, leading to a KeyError. - del IOLoop._ioloop_for_asyncio[self.asyncio_loop] - self.asyncio_loop.close() - - def add_handler(self, fd, handler, events): - fd, fileobj = self.split_fd(fd) - if fd in self.handlers: - raise ValueError("fd %s added twice" % fd) - self.handlers[fd] = (fileobj, stack_context.wrap(handler)) - if events & IOLoop.READ: - self.asyncio_loop.add_reader( - fd, self._handle_events, fd, IOLoop.READ) - self.readers.add(fd) - if events & IOLoop.WRITE: - self.asyncio_loop.add_writer( - fd, self._handle_events, fd, IOLoop.WRITE) - self.writers.add(fd) - - def update_handler(self, fd, events): - fd, fileobj = self.split_fd(fd) - if events & IOLoop.READ: - if fd not in self.readers: - self.asyncio_loop.add_reader( - fd, self._handle_events, fd, IOLoop.READ) - self.readers.add(fd) - else: - if fd in self.readers: - self.asyncio_loop.remove_reader(fd) - self.readers.remove(fd) - if events & IOLoop.WRITE: - if fd not in self.writers: - self.asyncio_loop.add_writer( - fd, self._handle_events, fd, IOLoop.WRITE) - self.writers.add(fd) - else: - if fd in self.writers: - self.asyncio_loop.remove_writer(fd) - self.writers.remove(fd) - - def remove_handler(self, fd): - fd, fileobj = self.split_fd(fd) - if fd not in self.handlers: - return - if fd in self.readers: - self.asyncio_loop.remove_reader(fd) - self.readers.remove(fd) - if fd in self.writers: - self.asyncio_loop.remove_writer(fd) - self.writers.remove(fd) - del self.handlers[fd] - - def _handle_events(self, fd, events): - fileobj, handler_func = self.handlers[fd] - handler_func(fileobj, events) - - def start(self): - try: - old_loop = asyncio.get_event_loop() - except (RuntimeError, AssertionError): - old_loop = None - try: - self._setup_logging() - asyncio.set_event_loop(self.asyncio_loop) - self.asyncio_loop.run_forever() - finally: - asyncio.set_event_loop(old_loop) - - def stop(self): - self.asyncio_loop.stop() - - def call_at(self, when, callback, *args, **kwargs): - # asyncio.call_at supports *args but not **kwargs, so bind them here. - # We do not synchronize self.time and asyncio_loop.time, so - # convert from absolute to relative. - return self.asyncio_loop.call_later( - max(0, when - self.time()), self._run_callback, - functools.partial(stack_context.wrap(callback), *args, **kwargs)) - - def remove_timeout(self, timeout): - timeout.cancel() - - def add_callback(self, callback, *args, **kwargs): - try: - self.asyncio_loop.call_soon_threadsafe( - self._run_callback, - functools.partial(stack_context.wrap(callback), *args, **kwargs)) - except RuntimeError: - # "Event loop is closed". Swallow the exception for - # consistency with PollIOLoop (and logical consistency - # with the fact that we can't guarantee that an - # add_callback that completes without error will - # eventually execute). - pass - - add_callback_from_signal = add_callback - - def run_in_executor(self, executor, func, *args): - return self.asyncio_loop.run_in_executor(executor, func, *args) - - def set_default_executor(self, executor): - return self.asyncio_loop.set_default_executor(executor) - - -class AsyncIOMainLoop(BaseAsyncIOLoop): - """``AsyncIOMainLoop`` creates an `.IOLoop` that corresponds to the - current ``asyncio`` event loop (i.e. the one returned by - ``asyncio.get_event_loop()``). - - .. deprecated:: 5.0 - - Now used automatically when appropriate; it is no longer necessary - to refer to this class directly. - - .. versionchanged:: 5.0 - - Closing an `AsyncIOMainLoop` now closes the underlying asyncio loop. - """ - def initialize(self, **kwargs): - super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), **kwargs) - - def make_current(self): - # AsyncIOMainLoop already refers to the current asyncio loop so - # nothing to do here. - pass - - -class AsyncIOLoop(BaseAsyncIOLoop): - """``AsyncIOLoop`` is an `.IOLoop` that runs on an ``asyncio`` event loop. - This class follows the usual Tornado semantics for creating new - ``IOLoops``; these loops are not necessarily related to the - ``asyncio`` default event loop. - - Each ``AsyncIOLoop`` creates a new ``asyncio.EventLoop``; this object - can be accessed with the ``asyncio_loop`` attribute. - - .. versionchanged:: 5.0 - - When an ``AsyncIOLoop`` becomes the current `.IOLoop`, it also sets - the current `asyncio` event loop. - - .. deprecated:: 5.0 - - Now used automatically when appropriate; it is no longer necessary - to refer to this class directly. - """ - def initialize(self, **kwargs): - self.is_current = False - loop = asyncio.new_event_loop() - try: - super(AsyncIOLoop, self).initialize(loop, **kwargs) - except Exception: - # If initialize() does not succeed (taking ownership of the loop), - # we have to close it. - loop.close() - raise - - def close(self, all_fds=False): - if self.is_current: - self.clear_current() - super(AsyncIOLoop, self).close(all_fds=all_fds) - - def make_current(self): - if not self.is_current: - try: - self.old_asyncio = asyncio.get_event_loop() - except (RuntimeError, AssertionError): - self.old_asyncio = None - self.is_current = True - asyncio.set_event_loop(self.asyncio_loop) - - def _clear_current_hook(self): - if self.is_current: - asyncio.set_event_loop(self.old_asyncio) - self.is_current = False - - -def to_tornado_future(asyncio_future): - """Convert an `asyncio.Future` to a `tornado.concurrent.Future`. - - .. versionadded:: 4.1 - - .. deprecated:: 5.0 - Tornado ``Futures`` have been merged with `asyncio.Future`, - so this method is now a no-op. - """ - return asyncio_future - - -def to_asyncio_future(tornado_future): - """Convert a Tornado yieldable object to an `asyncio.Future`. - - .. versionadded:: 4.1 - - .. versionchanged:: 4.3 - Now accepts any yieldable object, not just - `tornado.concurrent.Future`. - - .. deprecated:: 5.0 - Tornado ``Futures`` have been merged with `asyncio.Future`, - so this method is now equivalent to `tornado.gen.convert_yielded`. - """ - return convert_yielded(tornado_future) - - -class AnyThreadEventLoopPolicy(asyncio.DefaultEventLoopPolicy): - """Event loop policy that allows loop creation on any thread. - - The default `asyncio` event loop policy only automatically creates - event loops in the main threads. Other threads must create event - loops explicitly or `asyncio.get_event_loop` (and therefore - `.IOLoop.current`) will fail. Installing this policy allows event - loops to be created automatically on any thread, matching the - behavior of Tornado versions prior to 5.0 (or 5.0 on Python 2). - - Usage:: - - asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy()) - - .. versionadded:: 5.0 - - """ - def get_event_loop(self): - try: - return super().get_event_loop() - except (RuntimeError, AssertionError): - # This was an AssertionError in python 3.4.2 (which ships with debian jessie) - # and changed to a RuntimeError in 3.4.3. - # "There is no current event loop in thread %r" - loop = self.new_event_loop() - self.set_event_loop(loop) - return loop diff --git a/lib/tornado/platform/auto.py b/lib/tornado/platform/auto.py deleted file mode 100755 index 1a9133fa..00000000 --- a/lib/tornado/platform/auto.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Implementation of platform-specific functionality. - -For each function or class described in `tornado.platform.interface`, -the appropriate platform-specific implementation exists in this module. -Most code that needs access to this functionality should do e.g.:: - - from tornado.platform.auto import set_close_exec -""" - -from __future__ import absolute_import, division, print_function - -import os - -if 'APPENGINE_RUNTIME' in os.environ: - from tornado.platform.common import Waker - - def set_close_exec(fd): - pass -elif os.name == 'nt': - from tornado.platform.common import Waker - from tornado.platform.windows import set_close_exec -else: - from tornado.platform.posix import set_close_exec, Waker - -try: - # monotime monkey-patches the time module to have a monotonic function - # in versions of python before 3.3. - import monotime - # Silence pyflakes warning about this unused import - monotime -except ImportError: - pass -try: - # monotonic can provide a monotonic function in versions of python before - # 3.3, too. - from monotonic import monotonic as monotonic_time -except ImportError: - try: - from time import monotonic as monotonic_time - except ImportError: - monotonic_time = None - -__all__ = ['Waker', 'set_close_exec', 'monotonic_time'] diff --git a/lib/tornado/platform/auto.pyi b/lib/tornado/platform/auto.pyi deleted file mode 100755 index a1c97228..00000000 --- a/lib/tornado/platform/auto.pyi +++ /dev/null @@ -1,4 +0,0 @@ -# auto.py is full of patterns mypy doesn't like, so for type checking -# purposes we replace it with interface.py. - -from .interface import * diff --git a/lib/tornado/platform/caresresolver.py b/lib/tornado/platform/caresresolver.py deleted file mode 100755 index 768cb624..00000000 --- a/lib/tornado/platform/caresresolver.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import absolute_import, division, print_function -import pycares # type: ignore -import socket - -from tornado.concurrent import Future -from tornado import gen -from tornado.ioloop import IOLoop -from tornado.netutil import Resolver, is_valid_ip - - -class CaresResolver(Resolver): - """Name resolver based on the c-ares library. - - This is a non-blocking and non-threaded resolver. It may not produce - the same results as the system resolver, but can be used for non-blocking - resolution when threads cannot be used. - - c-ares fails to resolve some names when ``family`` is ``AF_UNSPEC``, - so it is only recommended for use in ``AF_INET`` (i.e. IPv4). This is - the default for ``tornado.simple_httpclient``, but other libraries - may default to ``AF_UNSPEC``. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - """ - def initialize(self): - self.io_loop = IOLoop.current() - self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb) - self.fds = {} - - def _sock_state_cb(self, fd, readable, writable): - state = ((IOLoop.READ if readable else 0) | - (IOLoop.WRITE if writable else 0)) - if not state: - self.io_loop.remove_handler(fd) - del self.fds[fd] - elif fd in self.fds: - self.io_loop.update_handler(fd, state) - self.fds[fd] = state - else: - self.io_loop.add_handler(fd, self._handle_events, state) - self.fds[fd] = state - - def _handle_events(self, fd, events): - read_fd = pycares.ARES_SOCKET_BAD - write_fd = pycares.ARES_SOCKET_BAD - if events & IOLoop.READ: - read_fd = fd - if events & IOLoop.WRITE: - write_fd = fd - self.channel.process_fd(read_fd, write_fd) - - @gen.coroutine - def resolve(self, host, port, family=0): - if is_valid_ip(host): - addresses = [host] - else: - # gethostbyname doesn't take callback as a kwarg - fut = Future() - self.channel.gethostbyname(host, family, - lambda result, error: fut.set_result((result, error))) - result, error = yield fut - if error: - raise IOError('C-Ares returned error %s: %s while resolving %s' % - (error, pycares.errno.strerror(error), host)) - addresses = result.addresses - addrinfo = [] - for address in addresses: - if '.' in address: - address_family = socket.AF_INET - elif ':' in address: - address_family = socket.AF_INET6 - else: - address_family = socket.AF_UNSPEC - if family != socket.AF_UNSPEC and family != address_family: - raise IOError('Requested socket family %d but got %d' % - (family, address_family)) - addrinfo.append((address_family, (address, port))) - raise gen.Return(addrinfo) diff --git a/lib/tornado/platform/common.py b/lib/tornado/platform/common.py deleted file mode 100755 index b597748d..00000000 --- a/lib/tornado/platform/common.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Lowest-common-denominator implementations of platform functionality.""" -from __future__ import absolute_import, division, print_function - -import errno -import socket -import time - -from tornado.platform import interface -from tornado.util import errno_from_exception - - -def try_close(f): - # Avoid issue #875 (race condition when using the file in another - # thread). - for i in range(10): - try: - f.close() - except IOError: - # Yield to another thread - time.sleep(1e-3) - else: - break - # Try a last time and let raise - f.close() - - -class Waker(interface.Waker): - """Create an OS independent asynchronous pipe. - - For use on platforms that don't have os.pipe() (or where pipes cannot - be passed to select()), but do have sockets. This includes Windows - and Jython. - """ - def __init__(self): - from .auto import set_close_exec - # Based on Zope select_trigger.py: - # https://github.com/zopefoundation/Zope/blob/master/src/ZServer/medusa/thread/select_trigger.py - - self.writer = socket.socket() - set_close_exec(self.writer.fileno()) - # Disable buffering -- pulling the trigger sends 1 byte, - # and we want that sent immediately, to wake up ASAP. - self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - count = 0 - while 1: - count += 1 - # Bind to a local port; for efficiency, let the OS pick - # a free port for us. - # Unfortunately, stress tests showed that we may not - # be able to connect to that port ("Address already in - # use") despite that the OS picked it. This appears - # to be a race bug in the Windows socket implementation. - # So we loop until a connect() succeeds (almost always - # on the first try). See the long thread at - # http://mail.zope.org/pipermail/zope/2005-July/160433.html - # for hideous details. - a = socket.socket() - set_close_exec(a.fileno()) - a.bind(("127.0.0.1", 0)) - a.listen(1) - connect_address = a.getsockname() # assigned (host, port) pair - try: - self.writer.connect(connect_address) - break # success - except socket.error as detail: - if (not hasattr(errno, 'WSAEADDRINUSE') or - errno_from_exception(detail) != errno.WSAEADDRINUSE): - # "Address already in use" is the only error - # I've seen on two WinXP Pro SP2 boxes, under - # Pythons 2.3.5 and 2.4.1. - raise - # (10048, 'Address already in use') - # assert count <= 2 # never triggered in Tim's tests - if count >= 10: # I've never seen it go above 2 - a.close() - self.writer.close() - raise socket.error("Cannot bind trigger!") - # Close `a` and try again. Note: I originally put a short - # sleep() here, but it didn't appear to help or hurt. - a.close() - - self.reader, addr = a.accept() - set_close_exec(self.reader.fileno()) - self.reader.setblocking(0) - self.writer.setblocking(0) - a.close() - self.reader_fd = self.reader.fileno() - - def fileno(self): - return self.reader.fileno() - - def write_fileno(self): - return self.writer.fileno() - - def wake(self): - try: - self.writer.send(b"x") - except (IOError, socket.error, ValueError): - pass - - def consume(self): - try: - while True: - result = self.reader.recv(1024) - if not result: - break - except (IOError, socket.error): - pass - - def close(self): - self.reader.close() - try_close(self.writer) diff --git a/lib/tornado/platform/epoll.py b/lib/tornado/platform/epoll.py deleted file mode 100755 index 4e346174..00000000 --- a/lib/tornado/platform/epoll.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""EPoll-based IOLoop implementation for Linux systems.""" -from __future__ import absolute_import, division, print_function - -import select - -from tornado.ioloop import PollIOLoop - - -class EPollIOLoop(PollIOLoop): - def initialize(self, **kwargs): - super(EPollIOLoop, self).initialize(impl=select.epoll(), **kwargs) diff --git a/lib/tornado/platform/interface.py b/lib/tornado/platform/interface.py deleted file mode 100755 index cac53264..00000000 --- a/lib/tornado/platform/interface.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Interfaces for platform-specific functionality. - -This module exists primarily for documentation purposes and as base classes -for other tornado.platform modules. Most code should import the appropriate -implementation from `tornado.platform.auto`. -""" - -from __future__ import absolute_import, division, print_function - - -def set_close_exec(fd): - """Sets the close-on-exec bit (``FD_CLOEXEC``)for a file descriptor.""" - raise NotImplementedError() - - -class Waker(object): - """A socket-like object that can wake another thread from ``select()``. - - The `~tornado.ioloop.IOLoop` will add the Waker's `fileno()` to - its ``select`` (or ``epoll`` or ``kqueue``) calls. When another - thread wants to wake up the loop, it calls `wake`. Once it has woken - up, it will call `consume` to do any necessary per-wake cleanup. When - the ``IOLoop`` is closed, it closes its waker too. - """ - def fileno(self): - """Returns the read file descriptor for this waker. - - Must be suitable for use with ``select()`` or equivalent on the - local platform. - """ - raise NotImplementedError() - - def write_fileno(self): - """Returns the write file descriptor for this waker.""" - raise NotImplementedError() - - def wake(self): - """Triggers activity on the waker's file descriptor.""" - raise NotImplementedError() - - def consume(self): - """Called after the listen has woken up to do any necessary cleanup.""" - raise NotImplementedError() - - def close(self): - """Closes the waker's file descriptor(s).""" - raise NotImplementedError() - - -def monotonic_time(): - raise NotImplementedError() diff --git a/lib/tornado/platform/kqueue.py b/lib/tornado/platform/kqueue.py deleted file mode 100755 index 4e0aee02..00000000 --- a/lib/tornado/platform/kqueue.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""KQueue-based IOLoop implementation for BSD/Mac systems.""" -from __future__ import absolute_import, division, print_function - -import select - -from tornado.ioloop import IOLoop, PollIOLoop - -assert hasattr(select, 'kqueue'), 'kqueue not supported' - - -class _KQueue(object): - """A kqueue-based event loop for BSD/Mac systems.""" - def __init__(self): - self._kqueue = select.kqueue() - self._active = {} - - def fileno(self): - return self._kqueue.fileno() - - def close(self): - self._kqueue.close() - - def register(self, fd, events): - if fd in self._active: - raise IOError("fd %s already registered" % fd) - self._control(fd, events, select.KQ_EV_ADD) - self._active[fd] = events - - def modify(self, fd, events): - self.unregister(fd) - self.register(fd, events) - - def unregister(self, fd): - events = self._active.pop(fd) - self._control(fd, events, select.KQ_EV_DELETE) - - def _control(self, fd, events, flags): - kevents = [] - if events & IOLoop.WRITE: - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_WRITE, flags=flags)) - if events & IOLoop.READ: - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_READ, flags=flags)) - # Even though control() takes a list, it seems to return EINVAL - # on Mac OS X (10.6) when there is more than one event in the list. - for kevent in kevents: - self._kqueue.control([kevent], 0) - - def poll(self, timeout): - kevents = self._kqueue.control(None, 1000, timeout) - events = {} - for kevent in kevents: - fd = kevent.ident - if kevent.filter == select.KQ_FILTER_READ: - events[fd] = events.get(fd, 0) | IOLoop.READ - if kevent.filter == select.KQ_FILTER_WRITE: - if kevent.flags & select.KQ_EV_EOF: - # If an asynchronous connection is refused, kqueue - # returns a write event with the EOF flag set. - # Turn this into an error for consistency with the - # other IOLoop implementations. - # Note that for read events, EOF may be returned before - # all data has been consumed from the socket buffer, - # so we only check for EOF on write events. - events[fd] = IOLoop.ERROR - else: - events[fd] = events.get(fd, 0) | IOLoop.WRITE - if kevent.flags & select.KQ_EV_ERROR: - events[fd] = events.get(fd, 0) | IOLoop.ERROR - return events.items() - - -class KQueueIOLoop(PollIOLoop): - def initialize(self, **kwargs): - super(KQueueIOLoop, self).initialize(impl=_KQueue(), **kwargs) diff --git a/lib/tornado/platform/posix.py b/lib/tornado/platform/posix.py deleted file mode 100755 index 6fe1fa83..00000000 --- a/lib/tornado/platform/posix.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Posix implementations of platform-specific functionality.""" - -from __future__ import absolute_import, division, print_function - -import fcntl -import os - -from tornado.platform import common, interface - - -def set_close_exec(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) - - -def _set_nonblocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - -class Waker(interface.Waker): - def __init__(self): - r, w = os.pipe() - _set_nonblocking(r) - _set_nonblocking(w) - set_close_exec(r) - set_close_exec(w) - self.reader = os.fdopen(r, "rb", 0) - self.writer = os.fdopen(w, "wb", 0) - - def fileno(self): - return self.reader.fileno() - - def write_fileno(self): - return self.writer.fileno() - - def wake(self): - try: - self.writer.write(b"x") - except (IOError, ValueError): - pass - - def consume(self): - try: - while True: - result = self.reader.read() - if not result: - break - except IOError: - pass - - def close(self): - self.reader.close() - common.try_close(self.writer) diff --git a/lib/tornado/platform/select.py b/lib/tornado/platform/select.py deleted file mode 100755 index 14e8a474..00000000 --- a/lib/tornado/platform/select.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright 2012 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Select-based IOLoop implementation. - -Used as a fallback for systems that don't support epoll or kqueue. -""" -from __future__ import absolute_import, division, print_function - -import select - -from tornado.ioloop import IOLoop, PollIOLoop - - -class _Select(object): - """A simple, select()-based IOLoop implementation for non-Linux systems""" - def __init__(self): - self.read_fds = set() - self.write_fds = set() - self.error_fds = set() - self.fd_sets = (self.read_fds, self.write_fds, self.error_fds) - - def close(self): - pass - - def register(self, fd, events): - if fd in self.read_fds or fd in self.write_fds or fd in self.error_fds: - raise IOError("fd %s already registered" % fd) - if events & IOLoop.READ: - self.read_fds.add(fd) - if events & IOLoop.WRITE: - self.write_fds.add(fd) - if events & IOLoop.ERROR: - self.error_fds.add(fd) - # Closed connections are reported as errors by epoll and kqueue, - # but as zero-byte reads by select, so when errors are requested - # we need to listen for both read and error. - # self.read_fds.add(fd) - - def modify(self, fd, events): - self.unregister(fd) - self.register(fd, events) - - def unregister(self, fd): - self.read_fds.discard(fd) - self.write_fds.discard(fd) - self.error_fds.discard(fd) - - def poll(self, timeout): - readable, writeable, errors = select.select( - self.read_fds, self.write_fds, self.error_fds, timeout) - events = {} - for fd in readable: - events[fd] = events.get(fd, 0) | IOLoop.READ - for fd in writeable: - events[fd] = events.get(fd, 0) | IOLoop.WRITE - for fd in errors: - events[fd] = events.get(fd, 0) | IOLoop.ERROR - return events.items() - - -class SelectIOLoop(PollIOLoop): - def initialize(self, **kwargs): - super(SelectIOLoop, self).initialize(impl=_Select(), **kwargs) diff --git a/lib/tornado/platform/twisted.py b/lib/tornado/platform/twisted.py deleted file mode 100755 index b38a755c..00000000 --- a/lib/tornado/platform/twisted.py +++ /dev/null @@ -1,609 +0,0 @@ -# Author: Ovidiu Predescu -# Date: July 2011 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Bridges between the Twisted reactor and Tornado IOLoop. - -This module lets you run applications and libraries written for -Twisted in a Tornado application. It can be used in two modes, -depending on which library's underlying event loop you want to use. - -This module has been tested with Twisted versions 11.0.0 and newer. -""" - -from __future__ import absolute_import, division, print_function - -import datetime -import functools -import numbers -import socket -import sys - -import twisted.internet.abstract # type: ignore -from twisted.internet.defer import Deferred # type: ignore -from twisted.internet.posixbase import PosixReactorBase # type: ignore -from twisted.internet.interfaces import IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor # type: ignore # noqa: E501 -from twisted.python import failure, log # type: ignore -from twisted.internet import error # type: ignore -import twisted.names.cache # type: ignore -import twisted.names.client # type: ignore -import twisted.names.hosts # type: ignore -import twisted.names.resolve # type: ignore - -from zope.interface import implementer # type: ignore - -from tornado.concurrent import Future, future_set_exc_info -from tornado.escape import utf8 -from tornado import gen -import tornado.ioloop -from tornado.log import app_log -from tornado.netutil import Resolver -from tornado.stack_context import NullContext, wrap -from tornado.ioloop import IOLoop -from tornado.util import timedelta_to_seconds - - -@implementer(IDelayedCall) -class TornadoDelayedCall(object): - """DelayedCall object for Tornado.""" - def __init__(self, reactor, seconds, f, *args, **kw): - self._reactor = reactor - self._func = functools.partial(f, *args, **kw) - self._time = self._reactor.seconds() + seconds - self._timeout = self._reactor._io_loop.add_timeout(self._time, - self._called) - self._active = True - - def _called(self): - self._active = False - self._reactor._removeDelayedCall(self) - try: - self._func() - except: - app_log.error("_called caught exception", exc_info=True) - - def getTime(self): - return self._time - - def cancel(self): - self._active = False - self._reactor._io_loop.remove_timeout(self._timeout) - self._reactor._removeDelayedCall(self) - - def delay(self, seconds): - self._reactor._io_loop.remove_timeout(self._timeout) - self._time += seconds - self._timeout = self._reactor._io_loop.add_timeout(self._time, - self._called) - - def reset(self, seconds): - self._reactor._io_loop.remove_timeout(self._timeout) - self._time = self._reactor.seconds() + seconds - self._timeout = self._reactor._io_loop.add_timeout(self._time, - self._called) - - def active(self): - return self._active - - -@implementer(IReactorTime, IReactorFDSet) -class TornadoReactor(PosixReactorBase): - """Twisted reactor built on the Tornado IOLoop. - - `TornadoReactor` implements the Twisted reactor interface on top of - the Tornado IOLoop. To use it, simply call `install` at the beginning - of the application:: - - import tornado.platform.twisted - tornado.platform.twisted.install() - from twisted.internet import reactor - - When the app is ready to start, call ``IOLoop.current().start()`` - instead of ``reactor.run()``. - - It is also possible to create a non-global reactor by calling - ``tornado.platform.twisted.TornadoReactor()``. However, if - the `.IOLoop` and reactor are to be short-lived (such as those used in - unit tests), additional cleanup may be required. Specifically, it is - recommended to call:: - - reactor.fireSystemEvent('shutdown') - reactor.disconnectAll() - - before closing the `.IOLoop`. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - .. deprecated:: 5.1 - - This class will be removed in Tornado 6.0. Use - ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` - instead. - - """ - def __init__(self): - self._io_loop = tornado.ioloop.IOLoop.current() - self._readers = {} # map of reader objects to fd - self._writers = {} # map of writer objects to fd - self._fds = {} # a map of fd to a (reader, writer) tuple - self._delayedCalls = {} - PosixReactorBase.__init__(self) - self.addSystemEventTrigger('during', 'shutdown', self.crash) - - # IOLoop.start() bypasses some of the reactor initialization. - # Fire off the necessary events if they weren't already triggered - # by reactor.run(). - def start_if_necessary(): - if not self._started: - self.fireSystemEvent('startup') - self._io_loop.add_callback(start_if_necessary) - - # IReactorTime - def seconds(self): - return self._io_loop.time() - - def callLater(self, seconds, f, *args, **kw): - dc = TornadoDelayedCall(self, seconds, f, *args, **kw) - self._delayedCalls[dc] = True - return dc - - def getDelayedCalls(self): - return [x for x in self._delayedCalls if x._active] - - def _removeDelayedCall(self, dc): - if dc in self._delayedCalls: - del self._delayedCalls[dc] - - # IReactorThreads - def callFromThread(self, f, *args, **kw): - assert callable(f), "%s is not callable" % f - with NullContext(): - # This NullContext is mainly for an edge case when running - # TwistedIOLoop on top of a TornadoReactor. - # TwistedIOLoop.add_callback uses reactor.callFromThread and - # should not pick up additional StackContexts along the way. - self._io_loop.add_callback(f, *args, **kw) - - # We don't need the waker code from the super class, Tornado uses - # its own waker. - def installWaker(self): - pass - - def wakeUp(self): - pass - - # IReactorFDSet - def _invoke_callback(self, fd, events): - if fd not in self._fds: - return - (reader, writer) = self._fds[fd] - if reader: - err = None - if reader.fileno() == -1: - err = error.ConnectionLost() - elif events & IOLoop.READ: - err = log.callWithLogger(reader, reader.doRead) - if err is None and events & IOLoop.ERROR: - err = error.ConnectionLost() - if err is not None: - self.removeReader(reader) - reader.readConnectionLost(failure.Failure(err)) - if writer: - err = None - if writer.fileno() == -1: - err = error.ConnectionLost() - elif events & IOLoop.WRITE: - err = log.callWithLogger(writer, writer.doWrite) - if err is None and events & IOLoop.ERROR: - err = error.ConnectionLost() - if err is not None: - self.removeWriter(writer) - writer.writeConnectionLost(failure.Failure(err)) - - def addReader(self, reader): - if reader in self._readers: - # Don't add the reader if it's already there - return - fd = reader.fileno() - self._readers[reader] = fd - if fd in self._fds: - (_, writer) = self._fds[fd] - self._fds[fd] = (reader, writer) - if writer: - # We already registered this fd for write events, - # update it for read events as well. - self._io_loop.update_handler(fd, IOLoop.READ | IOLoop.WRITE) - else: - with NullContext(): - self._fds[fd] = (reader, None) - self._io_loop.add_handler(fd, self._invoke_callback, - IOLoop.READ) - - def addWriter(self, writer): - if writer in self._writers: - return - fd = writer.fileno() - self._writers[writer] = fd - if fd in self._fds: - (reader, _) = self._fds[fd] - self._fds[fd] = (reader, writer) - if reader: - # We already registered this fd for read events, - # update it for write events as well. - self._io_loop.update_handler(fd, IOLoop.READ | IOLoop.WRITE) - else: - with NullContext(): - self._fds[fd] = (None, writer) - self._io_loop.add_handler(fd, self._invoke_callback, - IOLoop.WRITE) - - def removeReader(self, reader): - if reader in self._readers: - fd = self._readers.pop(reader) - (_, writer) = self._fds[fd] - if writer: - # We have a writer so we need to update the IOLoop for - # write events only. - self._fds[fd] = (None, writer) - self._io_loop.update_handler(fd, IOLoop.WRITE) - else: - # Since we have no writer registered, we remove the - # entry from _fds and unregister the handler from the - # IOLoop - del self._fds[fd] - self._io_loop.remove_handler(fd) - - def removeWriter(self, writer): - if writer in self._writers: - fd = self._writers.pop(writer) - (reader, _) = self._fds[fd] - if reader: - # We have a reader so we need to update the IOLoop for - # read events only. - self._fds[fd] = (reader, None) - self._io_loop.update_handler(fd, IOLoop.READ) - else: - # Since we have no reader registered, we remove the - # entry from the _fds and unregister the handler from - # the IOLoop. - del self._fds[fd] - self._io_loop.remove_handler(fd) - - def removeAll(self): - return self._removeAll(self._readers, self._writers) - - def getReaders(self): - return self._readers.keys() - - def getWriters(self): - return self._writers.keys() - - # The following functions are mainly used in twisted-style test cases; - # it is expected that most users of the TornadoReactor will call - # IOLoop.start() instead of Reactor.run(). - def stop(self): - PosixReactorBase.stop(self) - fire_shutdown = functools.partial(self.fireSystemEvent, "shutdown") - self._io_loop.add_callback(fire_shutdown) - - def crash(self): - PosixReactorBase.crash(self) - self._io_loop.stop() - - def doIteration(self, delay): - raise NotImplementedError("doIteration") - - def mainLoop(self): - # Since this class is intended to be used in applications - # where the top-level event loop is ``io_loop.start()`` rather - # than ``reactor.run()``, it is implemented a little - # differently than other Twisted reactors. We override - # ``mainLoop`` instead of ``doIteration`` and must implement - # timed call functionality on top of `.IOLoop.add_timeout` - # rather than using the implementation in - # ``PosixReactorBase``. - self._io_loop.start() - - -class _TestReactor(TornadoReactor): - """Subclass of TornadoReactor for use in unittests. - - This can't go in the test.py file because of import-order dependencies - with the Twisted reactor test builder. - """ - def __init__(self): - # always use a new ioloop - IOLoop.clear_current() - IOLoop(make_current=True) - super(_TestReactor, self).__init__() - IOLoop.clear_current() - - def listenTCP(self, port, factory, backlog=50, interface=''): - # default to localhost to avoid firewall prompts on the mac - if not interface: - interface = '127.0.0.1' - return super(_TestReactor, self).listenTCP( - port, factory, backlog=backlog, interface=interface) - - def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): - if not interface: - interface = '127.0.0.1' - return super(_TestReactor, self).listenUDP( - port, protocol, interface=interface, maxPacketSize=maxPacketSize) - - -def install(): - """Install this package as the default Twisted reactor. - - ``install()`` must be called very early in the startup process, - before most other twisted-related imports. Conversely, because it - initializes the `.IOLoop`, it cannot be called before - `.fork_processes` or multi-process `~.TCPServer.start`. These - conflicting requirements make it difficult to use `.TornadoReactor` - in multi-process mode, and an external process manager such as - ``supervisord`` is recommended instead. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - .. deprecated:: 5.1 - - This functio will be removed in Tornado 6.0. Use - ``twisted.internet.asyncioreactor.install`` instead. - """ - reactor = TornadoReactor() - from twisted.internet.main import installReactor # type: ignore - installReactor(reactor) - return reactor - - -@implementer(IReadDescriptor, IWriteDescriptor) -class _FD(object): - def __init__(self, fd, fileobj, handler): - self.fd = fd - self.fileobj = fileobj - self.handler = handler - self.reading = False - self.writing = False - self.lost = False - - def fileno(self): - return self.fd - - def doRead(self): - if not self.lost: - self.handler(self.fileobj, tornado.ioloop.IOLoop.READ) - - def doWrite(self): - if not self.lost: - self.handler(self.fileobj, tornado.ioloop.IOLoop.WRITE) - - def connectionLost(self, reason): - if not self.lost: - self.handler(self.fileobj, tornado.ioloop.IOLoop.ERROR) - self.lost = True - - writeConnectionLost = readConnectionLost = connectionLost - - def logPrefix(self): - return '' - - -class TwistedIOLoop(tornado.ioloop.IOLoop): - """IOLoop implementation that runs on Twisted. - - `TwistedIOLoop` implements the Tornado IOLoop interface on top of - the Twisted reactor. Recommended usage:: - - from tornado.platform.twisted import TwistedIOLoop - from twisted.internet import reactor - TwistedIOLoop().install() - # Set up your tornado application as usual using `IOLoop.instance` - reactor.run() - - Uses the global Twisted reactor by default. To create multiple - ``TwistedIOLoops`` in the same process, you must pass a unique reactor - when constructing each one. - - Not compatible with `tornado.process.Subprocess.set_exit_callback` - because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict - with each other. - - See also :meth:`tornado.ioloop.IOLoop.install` for general notes on - installing alternative IOLoops. - - .. deprecated:: 5.1 - - The `asyncio` event loop will be the only available implementation in - Tornado 6.0. - """ - def initialize(self, reactor=None, **kwargs): - super(TwistedIOLoop, self).initialize(**kwargs) - if reactor is None: - import twisted.internet.reactor # type: ignore - reactor = twisted.internet.reactor - self.reactor = reactor - self.fds = {} - - def close(self, all_fds=False): - fds = self.fds - self.reactor.removeAll() - for c in self.reactor.getDelayedCalls(): - c.cancel() - if all_fds: - for fd in fds.values(): - self.close_fd(fd.fileobj) - - def add_handler(self, fd, handler, events): - if fd in self.fds: - raise ValueError('fd %s added twice' % fd) - fd, fileobj = self.split_fd(fd) - self.fds[fd] = _FD(fd, fileobj, wrap(handler)) - if events & tornado.ioloop.IOLoop.READ: - self.fds[fd].reading = True - self.reactor.addReader(self.fds[fd]) - if events & tornado.ioloop.IOLoop.WRITE: - self.fds[fd].writing = True - self.reactor.addWriter(self.fds[fd]) - - def update_handler(self, fd, events): - fd, fileobj = self.split_fd(fd) - if events & tornado.ioloop.IOLoop.READ: - if not self.fds[fd].reading: - self.fds[fd].reading = True - self.reactor.addReader(self.fds[fd]) - else: - if self.fds[fd].reading: - self.fds[fd].reading = False - self.reactor.removeReader(self.fds[fd]) - if events & tornado.ioloop.IOLoop.WRITE: - if not self.fds[fd].writing: - self.fds[fd].writing = True - self.reactor.addWriter(self.fds[fd]) - else: - if self.fds[fd].writing: - self.fds[fd].writing = False - self.reactor.removeWriter(self.fds[fd]) - - def remove_handler(self, fd): - fd, fileobj = self.split_fd(fd) - if fd not in self.fds: - return - self.fds[fd].lost = True - if self.fds[fd].reading: - self.reactor.removeReader(self.fds[fd]) - if self.fds[fd].writing: - self.reactor.removeWriter(self.fds[fd]) - del self.fds[fd] - - def start(self): - old_current = IOLoop.current(instance=False) - try: - self._setup_logging() - self.make_current() - self.reactor.run() - finally: - if old_current is None: - IOLoop.clear_current() - else: - old_current.make_current() - - def stop(self): - self.reactor.crash() - - def add_timeout(self, deadline, callback, *args, **kwargs): - # This method could be simplified (since tornado 4.0) by - # overriding call_at instead of add_timeout, but we leave it - # for now as a test of backwards-compatibility. - if isinstance(deadline, numbers.Real): - delay = max(deadline - self.time(), 0) - elif isinstance(deadline, datetime.timedelta): - delay = timedelta_to_seconds(deadline) - else: - raise TypeError("Unsupported deadline %r") - return self.reactor.callLater( - delay, self._run_callback, - functools.partial(wrap(callback), *args, **kwargs)) - - def remove_timeout(self, timeout): - if timeout.active(): - timeout.cancel() - - def add_callback(self, callback, *args, **kwargs): - self.reactor.callFromThread( - self._run_callback, - functools.partial(wrap(callback), *args, **kwargs)) - - def add_callback_from_signal(self, callback, *args, **kwargs): - self.add_callback(callback, *args, **kwargs) - - -class TwistedResolver(Resolver): - """Twisted-based asynchronous resolver. - - This is a non-blocking and non-threaded resolver. It is - recommended only when threads cannot be used, since it has - limitations compared to the standard ``getaddrinfo``-based - `~tornado.netutil.Resolver` and - `~tornado.netutil.DefaultExecutorResolver`. Specifically, it returns at - most one result, and arguments other than ``host`` and ``family`` - are ignored. It may fail to resolve when ``family`` is not - ``socket.AF_UNSPEC``. - - Requires Twisted 12.1 or newer. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - """ - def initialize(self): - # partial copy of twisted.names.client.createResolver, which doesn't - # allow for a reactor to be passed in. - self.reactor = tornado.platform.twisted.TornadoReactor() - - host_resolver = twisted.names.hosts.Resolver('/etc/hosts') - cache_resolver = twisted.names.cache.CacheResolver(reactor=self.reactor) - real_resolver = twisted.names.client.Resolver('/etc/resolv.conf', - reactor=self.reactor) - self.resolver = twisted.names.resolve.ResolverChain( - [host_resolver, cache_resolver, real_resolver]) - - @gen.coroutine - def resolve(self, host, port, family=0): - # getHostByName doesn't accept IP addresses, so if the input - # looks like an IP address just return it immediately. - if twisted.internet.abstract.isIPAddress(host): - resolved = host - resolved_family = socket.AF_INET - elif twisted.internet.abstract.isIPv6Address(host): - resolved = host - resolved_family = socket.AF_INET6 - else: - deferred = self.resolver.getHostByName(utf8(host)) - fut = Future() - deferred.addBoth(fut.set_result) - resolved = yield fut - if isinstance(resolved, failure.Failure): - try: - resolved.raiseException() - except twisted.names.error.DomainError as e: - raise IOError(e) - elif twisted.internet.abstract.isIPAddress(resolved): - resolved_family = socket.AF_INET - elif twisted.internet.abstract.isIPv6Address(resolved): - resolved_family = socket.AF_INET6 - else: - resolved_family = socket.AF_UNSPEC - if family != socket.AF_UNSPEC and family != resolved_family: - raise Exception('Requested socket family %d but got %d' % - (family, resolved_family)) - result = [ - (resolved_family, (resolved, port)), - ] - raise gen.Return(result) - - -if hasattr(gen.convert_yielded, 'register'): - @gen.convert_yielded.register(Deferred) # type: ignore - def _(d): - f = Future() - - def errback(failure): - try: - failure.raiseException() - # Should never happen, but just in case - raise Exception("errback called without error") - except: - future_set_exc_info(f, sys.exc_info()) - d.addCallbacks(f.set_result, errback) - return f diff --git a/lib/tornado/platform/windows.py b/lib/tornado/platform/windows.py deleted file mode 100755 index 41277006..00000000 --- a/lib/tornado/platform/windows.py +++ /dev/null @@ -1,20 +0,0 @@ -# NOTE: win32 support is currently experimental, and not recommended -# for production use. - - -from __future__ import absolute_import, division, print_function -import ctypes # type: ignore -import ctypes.wintypes # type: ignore - -# See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx -SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation -SetHandleInformation.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD, ctypes.wintypes.DWORD) # noqa: E501 -SetHandleInformation.restype = ctypes.wintypes.BOOL - -HANDLE_FLAG_INHERIT = 0x00000001 - - -def set_close_exec(fd): - success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, 0) - if not success: - raise ctypes.WinError() diff --git a/lib/tornado/process.py b/lib/tornado/process.py deleted file mode 100755 index 122fd7e1..00000000 --- a/lib/tornado/process.py +++ /dev/null @@ -1,361 +0,0 @@ -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utilities for working with multiple processes, including both forking -the server into multiple processes and managing subprocesses. -""" - -from __future__ import absolute_import, division, print_function - -import errno -import os -import signal -import subprocess -import sys -import time - -from binascii import hexlify - -from tornado.concurrent import Future, future_set_result_unless_cancelled -from tornado import ioloop -from tornado.iostream import PipeIOStream -from tornado.log import gen_log -from tornado.platform.auto import set_close_exec -from tornado import stack_context -from tornado.util import errno_from_exception, PY3 - -try: - import multiprocessing -except ImportError: - # Multiprocessing is not available on Google App Engine. - multiprocessing = None - -if PY3: - long = int - -# Re-export this exception for convenience. -try: - CalledProcessError = subprocess.CalledProcessError -except AttributeError: - # The subprocess module exists in Google App Engine, but is empty. - # This module isn't very useful in that case, but it should - # at least be importable. - if 'APPENGINE_RUNTIME' not in os.environ: - raise - - -def cpu_count(): - """Returns the number of processors on this machine.""" - if multiprocessing is None: - return 1 - try: - return multiprocessing.cpu_count() - except NotImplementedError: - pass - try: - return os.sysconf("SC_NPROCESSORS_CONF") - except (AttributeError, ValueError): - pass - gen_log.error("Could not detect number of processors; assuming 1") - return 1 - - -def _reseed_random(): - if 'random' not in sys.modules: - return - import random - # If os.urandom is available, this method does the same thing as - # random.seed (at least as of python 2.6). If os.urandom is not - # available, we mix in the pid in addition to a timestamp. - try: - seed = long(hexlify(os.urandom(16)), 16) - except NotImplementedError: - seed = int(time.time() * 1000) ^ os.getpid() - random.seed(seed) - - -def _pipe_cloexec(): - r, w = os.pipe() - set_close_exec(r) - set_close_exec(w) - return r, w - - -_task_id = None - - -def fork_processes(num_processes, max_restarts=100): - """Starts multiple worker processes. - - If ``num_processes`` is None or <= 0, we detect the number of cores - available on this machine and fork that number of child - processes. If ``num_processes`` is given and > 0, we fork that - specific number of sub-processes. - - Since we use processes and not threads, there is no shared memory - between any server code. - - Note that multiple processes are not compatible with the autoreload - module (or the ``autoreload=True`` option to `tornado.web.Application` - which defaults to True when ``debug=True``). - When using multiple processes, no IOLoops can be created or - referenced until after the call to ``fork_processes``. - - In each child process, ``fork_processes`` returns its *task id*, a - number between 0 and ``num_processes``. Processes that exit - abnormally (due to a signal or non-zero exit status) are restarted - with the same id (up to ``max_restarts`` times). In the parent - process, ``fork_processes`` returns None if all child processes - have exited normally, but will otherwise only exit by throwing an - exception. - """ - global _task_id - assert _task_id is None - if num_processes is None or num_processes <= 0: - num_processes = cpu_count() - gen_log.info("Starting %d processes", num_processes) - children = {} - - def start_child(i): - pid = os.fork() - if pid == 0: - # child process - _reseed_random() - global _task_id - _task_id = i - return i - else: - children[pid] = i - return None - - for i in range(num_processes): - id = start_child(i) - if id is not None: - return id - num_restarts = 0 - while children: - try: - pid, status = os.wait() - except OSError as e: - if errno_from_exception(e) == errno.EINTR: - continue - raise - if pid not in children: - continue - id = children.pop(pid) - if os.WIFSIGNALED(status): - gen_log.warning("child %d (pid %d) killed by signal %d, restarting", - id, pid, os.WTERMSIG(status)) - elif os.WEXITSTATUS(status) != 0: - gen_log.warning("child %d (pid %d) exited with status %d, restarting", - id, pid, os.WEXITSTATUS(status)) - else: - gen_log.info("child %d (pid %d) exited normally", id, pid) - continue - num_restarts += 1 - if num_restarts > max_restarts: - raise RuntimeError("Too many child restarts, giving up") - new_id = start_child(id) - if new_id is not None: - return new_id - # All child processes exited cleanly, so exit the master process - # instead of just returning to right after the call to - # fork_processes (which will probably just start up another IOLoop - # unless the caller checks the return value). - sys.exit(0) - - -def task_id(): - """Returns the current task id, if any. - - Returns None if this process was not created by `fork_processes`. - """ - global _task_id - return _task_id - - -class Subprocess(object): - """Wraps ``subprocess.Popen`` with IOStream support. - - The constructor is the same as ``subprocess.Popen`` with the following - additions: - - * ``stdin``, ``stdout``, and ``stderr`` may have the value - ``tornado.process.Subprocess.STREAM``, which will make the corresponding - attribute of the resulting Subprocess a `.PipeIOStream`. If this option - is used, the caller is responsible for closing the streams when done - with them. - - The ``Subprocess.STREAM`` option and the ``set_exit_callback`` and - ``wait_for_exit`` methods do not work on Windows. There is - therefore no reason to use this class instead of - ``subprocess.Popen`` on that platform. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - - """ - STREAM = object() - - _initialized = False - _waiting = {} # type: ignore - - def __init__(self, *args, **kwargs): - self.io_loop = ioloop.IOLoop.current() - # All FDs we create should be closed on error; those in to_close - # should be closed in the parent process on success. - pipe_fds = [] - to_close = [] - if kwargs.get('stdin') is Subprocess.STREAM: - in_r, in_w = _pipe_cloexec() - kwargs['stdin'] = in_r - pipe_fds.extend((in_r, in_w)) - to_close.append(in_r) - self.stdin = PipeIOStream(in_w) - if kwargs.get('stdout') is Subprocess.STREAM: - out_r, out_w = _pipe_cloexec() - kwargs['stdout'] = out_w - pipe_fds.extend((out_r, out_w)) - to_close.append(out_w) - self.stdout = PipeIOStream(out_r) - if kwargs.get('stderr') is Subprocess.STREAM: - err_r, err_w = _pipe_cloexec() - kwargs['stderr'] = err_w - pipe_fds.extend((err_r, err_w)) - to_close.append(err_w) - self.stderr = PipeIOStream(err_r) - try: - self.proc = subprocess.Popen(*args, **kwargs) - except: - for fd in pipe_fds: - os.close(fd) - raise - for fd in to_close: - os.close(fd) - for attr in ['stdin', 'stdout', 'stderr', 'pid']: - if not hasattr(self, attr): # don't clobber streams set above - setattr(self, attr, getattr(self.proc, attr)) - self._exit_callback = None - self.returncode = None - - def set_exit_callback(self, callback): - """Runs ``callback`` when this process exits. - - The callback takes one argument, the return code of the process. - - This method uses a ``SIGCHLD`` handler, which is a global setting - and may conflict if you have other libraries trying to handle the - same signal. If you are using more than one ``IOLoop`` it may - be necessary to call `Subprocess.initialize` first to designate - one ``IOLoop`` to run the signal handlers. - - In many cases a close callback on the stdout or stderr streams - can be used as an alternative to an exit callback if the - signal handler is causing a problem. - """ - self._exit_callback = stack_context.wrap(callback) - Subprocess.initialize() - Subprocess._waiting[self.pid] = self - Subprocess._try_cleanup_process(self.pid) - - def wait_for_exit(self, raise_error=True): - """Returns a `.Future` which resolves when the process exits. - - Usage:: - - ret = yield proc.wait_for_exit() - - This is a coroutine-friendly alternative to `set_exit_callback` - (and a replacement for the blocking `subprocess.Popen.wait`). - - By default, raises `subprocess.CalledProcessError` if the process - has a non-zero exit status. Use ``wait_for_exit(raise_error=False)`` - to suppress this behavior and return the exit status without raising. - - .. versionadded:: 4.2 - """ - future = Future() - - def callback(ret): - if ret != 0 and raise_error: - # Unfortunately we don't have the original args any more. - future.set_exception(CalledProcessError(ret, None)) - else: - future_set_result_unless_cancelled(future, ret) - self.set_exit_callback(callback) - return future - - @classmethod - def initialize(cls): - """Initializes the ``SIGCHLD`` handler. - - The signal handler is run on an `.IOLoop` to avoid locking issues. - Note that the `.IOLoop` used for signal handling need not be the - same one used by individual Subprocess objects (as long as the - ``IOLoops`` are each running in separate threads). - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been - removed. - """ - if cls._initialized: - return - io_loop = ioloop.IOLoop.current() - cls._old_sigchld = signal.signal( - signal.SIGCHLD, - lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup)) - cls._initialized = True - - @classmethod - def uninitialize(cls): - """Removes the ``SIGCHLD`` handler.""" - if not cls._initialized: - return - signal.signal(signal.SIGCHLD, cls._old_sigchld) - cls._initialized = False - - @classmethod - def _cleanup(cls): - for pid in list(cls._waiting.keys()): # make a copy - cls._try_cleanup_process(pid) - - @classmethod - def _try_cleanup_process(cls, pid): - try: - ret_pid, status = os.waitpid(pid, os.WNOHANG) - except OSError as e: - if errno_from_exception(e) == errno.ECHILD: - return - if ret_pid == 0: - return - assert ret_pid == pid - subproc = cls._waiting.pop(pid) - subproc.io_loop.add_callback_from_signal( - subproc._set_returncode, status) - - def _set_returncode(self, status): - if os.WIFSIGNALED(status): - self.returncode = -os.WTERMSIG(status) - else: - assert os.WIFEXITED(status) - self.returncode = os.WEXITSTATUS(status) - # We've taken over wait() duty from the subprocess.Popen - # object. If we don't inform it of the process's return code, - # it will log a warning at destruction in python 3.6+. - self.proc.returncode = self.returncode - if self._exit_callback: - callback = self._exit_callback - self._exit_callback = None - callback(self.returncode) diff --git a/lib/tornado/queues.py b/lib/tornado/queues.py deleted file mode 100755 index 7cb96bfc..00000000 --- a/lib/tornado/queues.py +++ /dev/null @@ -1,379 +0,0 @@ -# Copyright 2015 The Tornado Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Asynchronous queues for coroutines. These classes are very similar -to those provided in the standard library's `asyncio package -<https://docs.python.org/3/library/asyncio-queue.html>`_. - -.. warning:: - - Unlike the standard library's `queue` module, the classes defined here - are *not* thread-safe. To use these queues from another thread, - use `.IOLoop.add_callback` to transfer control to the `.IOLoop` thread - before calling any queue methods. - -""" - -from __future__ import absolute_import, division, print_function - -import collections -import heapq - -from tornado import gen, ioloop -from tornado.concurrent import Future, future_set_result_unless_cancelled -from tornado.locks import Event - -__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty'] - - -class QueueEmpty(Exception): - """Raised by `.Queue.get_nowait` when the queue has no items.""" - pass - - -class QueueFull(Exception): - """Raised by `.Queue.put_nowait` when a queue is at its maximum size.""" - pass - - -def _set_timeout(future, timeout): - if timeout: - def on_timeout(): - if not future.done(): - future.set_exception(gen.TimeoutError()) - io_loop = ioloop.IOLoop.current() - timeout_handle = io_loop.add_timeout(timeout, on_timeout) - future.add_done_callback( - lambda _: io_loop.remove_timeout(timeout_handle)) - - -class _QueueIterator(object): - def __init__(self, q): - self.q = q - - def __anext__(self): - return self.q.get() - - -class Queue(object): - """Coordinate producer and consumer coroutines. - - If maxsize is 0 (the default) the queue size is unbounded. - - .. testcode:: - - from tornado import gen - from tornado.ioloop import IOLoop - from tornado.queues import Queue - - q = Queue(maxsize=2) - - async def consumer(): - async for item in q: - try: - print('Doing work on %s' % item) - await gen.sleep(0.01) - finally: - q.task_done() - - async def producer(): - for item in range(5): - await q.put(item) - print('Put %s' % item) - - async def main(): - # Start consumer without waiting (since it never finishes). - IOLoop.current().spawn_callback(consumer) - await producer() # Wait for producer to put all tasks. - await q.join() # Wait for consumer to finish all tasks. - print('Done') - - IOLoop.current().run_sync(main) - - .. testoutput:: - - Put 0 - Put 1 - Doing work on 0 - Put 2 - Doing work on 1 - Put 3 - Doing work on 2 - Put 4 - Doing work on 3 - Doing work on 4 - Done - - - In versions of Python without native coroutines (before 3.5), - ``consumer()`` could be written as:: - - @gen.coroutine - def consumer(): - while True: - item = yield q.get() - try: - print('Doing work on %s' % item) - yield gen.sleep(0.01) - finally: - q.task_done() - - .. versionchanged:: 4.3 - Added ``async for`` support in Python 3.5. - - """ - def __init__(self, maxsize=0): - if maxsize is None: - raise TypeError("maxsize can't be None") - - if maxsize < 0: - raise ValueError("maxsize can't be negative") - - self._maxsize = maxsize - self._init() - self._getters = collections.deque([]) # Futures. - self._putters = collections.deque([]) # Pairs of (item, Future). - self._unfinished_tasks = 0 - self._finished = Event() - self._finished.set() - - @property - def maxsize(self): - """Number of items allowed in the queue.""" - return self._maxsize - - def qsize(self): - """Number of items in the queue.""" - return len(self._queue) - - def empty(self): - return not self._queue - - def full(self): - if self.maxsize == 0: - return False - else: - return self.qsize() >= self.maxsize - - def put(self, item, timeout=None): - """Put an item into the queue, perhaps waiting until there is room. - - Returns a Future, which raises `tornado.util.TimeoutError` after a - timeout. - - ``timeout`` may be a number denoting a time (on the same - scale as `tornado.ioloop.IOLoop.time`, normally `time.time`), or a - `datetime.timedelta` object for a deadline relative to the - current time. - """ - future = Future() - try: - self.put_nowait(item) - except QueueFull: - self._putters.append((item, future)) - _set_timeout(future, timeout) - else: - future.set_result(None) - return future - - def put_nowait(self, item): - """Put an item into the queue without blocking. - - If no free slot is immediately available, raise `QueueFull`. - """ - self._consume_expired() - if self._getters: - assert self.empty(), "queue non-empty, why are getters waiting?" - getter = self._getters.popleft() - self.__put_internal(item) - future_set_result_unless_cancelled(getter, self._get()) - elif self.full(): - raise QueueFull - else: - self.__put_internal(item) - - def get(self, timeout=None): - """Remove and return an item from the queue. - - Returns a Future which resolves once an item is available, or raises - `tornado.util.TimeoutError` after a timeout. - - ``timeout`` may be a number denoting a time (on the same - scale as `tornado.ioloop.IOLoop.time`, normally `time.time`), or a - `datetime.timedelta` object for a deadline relative to the - current time. - """ - future = Future() - try: - future.set_result(self.get_nowait()) - except QueueEmpty: - self._getters.append(future) - _set_timeout(future, timeout) - return future - - def get_nowait(self): - """Remove and return an item from the queue without blocking. - - Return an item if one is immediately available, else raise - `QueueEmpty`. - """ - self._consume_expired() - if self._putters: - assert self.full(), "queue not full, why are putters waiting?" - item, putter = self._putters.popleft() - self.__put_internal(item) - future_set_result_unless_cancelled(putter, None) - return self._get() - elif self.qsize(): - return self._get() - else: - raise QueueEmpty - - def task_done(self): - """Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each `.get` used to fetch a task, a - subsequent call to `.task_done` tells the queue that the processing - on the task is complete. - - If a `.join` is blocking, it resumes when all items have been - processed; that is, when every `.put` is matched by a `.task_done`. - - Raises `ValueError` if called more times than `.put`. - """ - if self._unfinished_tasks <= 0: - raise ValueError('task_done() called too many times') - self._unfinished_tasks -= 1 - if self._unfinished_tasks == 0: - self._finished.set() - - def join(self, timeout=None): - """Block until all items in the queue are processed. - - Returns a Future, which raises `tornado.util.TimeoutError` after a - timeout. - """ - return self._finished.wait(timeout) - - def __aiter__(self): - return _QueueIterator(self) - - # These three are overridable in subclasses. - def _init(self): - self._queue = collections.deque() - - def _get(self): - return self._queue.popleft() - - def _put(self, item): - self._queue.append(item) - # End of the overridable methods. - - def __put_internal(self, item): - self._unfinished_tasks += 1 - self._finished.clear() - self._put(item) - - def _consume_expired(self): - # Remove timed-out waiters. - while self._putters and self._putters[0][1].done(): - self._putters.popleft() - - while self._getters and self._getters[0].done(): - self._getters.popleft() - - def __repr__(self): - return '<%s at %s %s>' % ( - type(self).__name__, hex(id(self)), self._format()) - - def __str__(self): - return '<%s %s>' % (type(self).__name__, self._format()) - - def _format(self): - result = 'maxsize=%r' % (self.maxsize, ) - if getattr(self, '_queue', None): - result += ' queue=%r' % self._queue - if self._getters: - result += ' getters[%s]' % len(self._getters) - if self._putters: - result += ' putters[%s]' % len(self._putters) - if self._unfinished_tasks: - result += ' tasks=%s' % self._unfinished_tasks - return result - - -class PriorityQueue(Queue): - """A `.Queue` that retrieves entries in priority order, lowest first. - - Entries are typically tuples like ``(priority number, data)``. - - .. testcode:: - - from tornado.queues import PriorityQueue - - q = PriorityQueue() - q.put((1, 'medium-priority item')) - q.put((0, 'high-priority item')) - q.put((10, 'low-priority item')) - - print(q.get_nowait()) - print(q.get_nowait()) - print(q.get_nowait()) - - .. testoutput:: - - (0, 'high-priority item') - (1, 'medium-priority item') - (10, 'low-priority item') - """ - def _init(self): - self._queue = [] - - def _put(self, item): - heapq.heappush(self._queue, item) - - def _get(self): - return heapq.heappop(self._queue) - - -class LifoQueue(Queue): - """A `.Queue` that retrieves the most recently put items first. - - .. testcode:: - - from tornado.queues import LifoQueue - - q = LifoQueue() - q.put(3) - q.put(2) - q.put(1) - - print(q.get_nowait()) - print(q.get_nowait()) - print(q.get_nowait()) - - .. testoutput:: - - 1 - 2 - 3 - """ - def _init(self): - self._queue = [] - - def _put(self, item): - self._queue.append(item) - - def _get(self): - return self._queue.pop() diff --git a/lib/tornado/routing.py b/lib/tornado/routing.py deleted file mode 100755 index e56d1a75..00000000 --- a/lib/tornado/routing.py +++ /dev/null @@ -1,641 +0,0 @@ -# Copyright 2015 The Tornado Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Flexible routing implementation. - -Tornado routes HTTP requests to appropriate handlers using `Router` -class implementations. The `tornado.web.Application` class is a -`Router` implementation and may be used directly, or the classes in -this module may be used for additional flexibility. The `RuleRouter` -class can match on more criteria than `.Application`, or the `Router` -interface can be subclassed for maximum customization. - -`Router` interface extends `~.httputil.HTTPServerConnectionDelegate` -to provide additional routing capabilities. This also means that any -`Router` implementation can be used directly as a ``request_callback`` -for `~.httpserver.HTTPServer` constructor. - -`Router` subclass must implement a ``find_handler`` method to provide -a suitable `~.httputil.HTTPMessageDelegate` instance to handle the -request: - -.. code-block:: python - - class CustomRouter(Router): - def find_handler(self, request, **kwargs): - # some routing logic providing a suitable HTTPMessageDelegate instance - return MessageDelegate(request.connection) - - class MessageDelegate(HTTPMessageDelegate): - def __init__(self, connection): - self.connection = connection - - def finish(self): - self.connection.write_headers( - ResponseStartLine("HTTP/1.1", 200, "OK"), - HTTPHeaders({"Content-Length": "2"}), - b"OK") - self.connection.finish() - - router = CustomRouter() - server = HTTPServer(router) - -The main responsibility of `Router` implementation is to provide a -mapping from a request to `~.httputil.HTTPMessageDelegate` instance -that will handle this request. In the example above we can see that -routing is possible even without instantiating an `~.web.Application`. - -For routing to `~.web.RequestHandler` implementations we need an -`~.web.Application` instance. `~.web.Application.get_handler_delegate` -provides a convenient way to create `~.httputil.HTTPMessageDelegate` -for a given request and `~.web.RequestHandler`. - -Here is a simple example of how we can we route to -`~.web.RequestHandler` subclasses by HTTP method: - -.. code-block:: python - - resources = {} - - class GetResource(RequestHandler): - def get(self, path): - if path not in resources: - raise HTTPError(404) - - self.finish(resources[path]) - - class PostResource(RequestHandler): - def post(self, path): - resources[path] = self.request.body - - class HTTPMethodRouter(Router): - def __init__(self, app): - self.app = app - - def find_handler(self, request, **kwargs): - handler = GetResource if request.method == "GET" else PostResource - return self.app.get_handler_delegate(request, handler, path_args=[request.path]) - - router = HTTPMethodRouter(Application()) - server = HTTPServer(router) - -`ReversibleRouter` interface adds the ability to distinguish between -the routes and reverse them to the original urls using route's name -and additional arguments. `~.web.Application` is itself an -implementation of `ReversibleRouter` class. - -`RuleRouter` and `ReversibleRuleRouter` are implementations of -`Router` and `ReversibleRouter` interfaces and can be used for -creating rule-based routing configurations. - -Rules are instances of `Rule` class. They contain a `Matcher`, which -provides the logic for determining whether the rule is a match for a -particular request and a target, which can be one of the following. - -1) An instance of `~.httputil.HTTPServerConnectionDelegate`: - -.. code-block:: python - - router = RuleRouter([ - Rule(PathMatches("/handler"), ConnectionDelegate()), - # ... more rules - ]) - - class ConnectionDelegate(HTTPServerConnectionDelegate): - def start_request(self, server_conn, request_conn): - return MessageDelegate(request_conn) - -2) A callable accepting a single argument of `~.httputil.HTTPServerRequest` type: - -.. code-block:: python - - router = RuleRouter([ - Rule(PathMatches("/callable"), request_callable) - ]) - - def request_callable(request): - request.write(b"HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\n\\r\\nOK") - request.finish() - -3) Another `Router` instance: - -.. code-block:: python - - router = RuleRouter([ - Rule(PathMatches("/router.*"), CustomRouter()) - ]) - -Of course a nested `RuleRouter` or a `~.web.Application` is allowed: - -.. code-block:: python - - router = RuleRouter([ - Rule(HostMatches("example.com"), RuleRouter([ - Rule(PathMatches("/app1/.*"), Application([(r"/app1/handler", Handler)]))), - ])) - ]) - - server = HTTPServer(router) - -In the example below `RuleRouter` is used to route between applications: - -.. code-block:: python - - app1 = Application([ - (r"/app1/handler", Handler1), - # other handlers ... - ]) - - app2 = Application([ - (r"/app2/handler", Handler2), - # other handlers ... - ]) - - router = RuleRouter([ - Rule(PathMatches("/app1.*"), app1), - Rule(PathMatches("/app2.*"), app2) - ]) - - server = HTTPServer(router) - -For more information on application-level routing see docs for `~.web.Application`. - -.. versionadded:: 4.5 - -""" - -from __future__ import absolute_import, division, print_function - -import re -from functools import partial - -from tornado import httputil -from tornado.httpserver import _CallableAdapter -from tornado.escape import url_escape, url_unescape, utf8 -from tornado.log import app_log -from tornado.util import basestring_type, import_object, re_unescape, unicode_type - -try: - import typing # noqa -except ImportError: - pass - - -class Router(httputil.HTTPServerConnectionDelegate): - """Abstract router interface.""" - - def find_handler(self, request, **kwargs): - # type: (httputil.HTTPServerRequest, typing.Any)->httputil.HTTPMessageDelegate - """Must be implemented to return an appropriate instance of `~.httputil.HTTPMessageDelegate` - that can serve the request. - Routing implementations may pass additional kwargs to extend the routing logic. - - :arg httputil.HTTPServerRequest request: current HTTP request. - :arg kwargs: additional keyword arguments passed by routing implementation. - :returns: an instance of `~.httputil.HTTPMessageDelegate` that will be used to - process the request. - """ - raise NotImplementedError() - - def start_request(self, server_conn, request_conn): - return _RoutingDelegate(self, server_conn, request_conn) - - -class ReversibleRouter(Router): - """Abstract router interface for routers that can handle named routes - and support reversing them to original urls. - """ - - def reverse_url(self, name, *args): - """Returns url string for a given route name and arguments - or ``None`` if no match is found. - - :arg str name: route name. - :arg args: url parameters. - :returns: parametrized url string for a given route name (or ``None``). - """ - raise NotImplementedError() - - -class _RoutingDelegate(httputil.HTTPMessageDelegate): - def __init__(self, router, server_conn, request_conn): - self.server_conn = server_conn - self.request_conn = request_conn - self.delegate = None - self.router = router # type: Router - - def headers_received(self, start_line, headers): - request = httputil.HTTPServerRequest( - connection=self.request_conn, - server_connection=self.server_conn, - start_line=start_line, headers=headers) - - self.delegate = self.router.find_handler(request) - if self.delegate is None: - app_log.debug("Delegate for %s %s request not found", - start_line.method, start_line.path) - self.delegate = _DefaultMessageDelegate(self.request_conn) - - return self.delegate.headers_received(start_line, headers) - - def data_received(self, chunk): - return self.delegate.data_received(chunk) - - def finish(self): - self.delegate.finish() - - def on_connection_close(self): - self.delegate.on_connection_close() - - -class _DefaultMessageDelegate(httputil.HTTPMessageDelegate): - def __init__(self, connection): - self.connection = connection - - def finish(self): - self.connection.write_headers( - httputil.ResponseStartLine("HTTP/1.1", 404, "Not Found"), httputil.HTTPHeaders()) - self.connection.finish() - - -class RuleRouter(Router): - """Rule-based router implementation.""" - - def __init__(self, rules=None): - """Constructs a router from an ordered list of rules:: - - RuleRouter([ - Rule(PathMatches("/handler"), Target), - # ... more rules - ]) - - You can also omit explicit `Rule` constructor and use tuples of arguments:: - - RuleRouter([ - (PathMatches("/handler"), Target), - ]) - - `PathMatches` is a default matcher, so the example above can be simplified:: - - RuleRouter([ - ("/handler", Target), - ]) - - In the examples above, ``Target`` can be a nested `Router` instance, an instance of - `~.httputil.HTTPServerConnectionDelegate` or an old-style callable, - accepting a request argument. - - :arg rules: a list of `Rule` instances or tuples of `Rule` - constructor arguments. - """ - self.rules = [] # type: typing.List[Rule] - if rules: - self.add_rules(rules) - - def add_rules(self, rules): - """Appends new rules to the router. - - :arg rules: a list of Rule instances (or tuples of arguments, which are - passed to Rule constructor). - """ - for rule in rules: - if isinstance(rule, (tuple, list)): - assert len(rule) in (2, 3, 4) - if isinstance(rule[0], basestring_type): - rule = Rule(PathMatches(rule[0]), *rule[1:]) - else: - rule = Rule(*rule) - - self.rules.append(self.process_rule(rule)) - - def process_rule(self, rule): - """Override this method for additional preprocessing of each rule. - - :arg Rule rule: a rule to be processed. - :returns: the same or modified Rule instance. - """ - return rule - - def find_handler(self, request, **kwargs): - for rule in self.rules: - target_params = rule.matcher.match(request) - if target_params is not None: - if rule.target_kwargs: - target_params['target_kwargs'] = rule.target_kwargs - - delegate = self.get_target_delegate( - rule.target, request, **target_params) - - if delegate is not None: - return delegate - - return None - - def get_target_delegate(self, target, request, **target_params): - """Returns an instance of `~.httputil.HTTPMessageDelegate` for a - Rule's target. This method is called by `~.find_handler` and can be - extended to provide additional target types. - - :arg target: a Rule's target. - :arg httputil.HTTPServerRequest request: current request. - :arg target_params: additional parameters that can be useful - for `~.httputil.HTTPMessageDelegate` creation. - """ - if isinstance(target, Router): - return target.find_handler(request, **target_params) - - elif isinstance(target, httputil.HTTPServerConnectionDelegate): - return target.start_request(request.server_connection, request.connection) - - elif callable(target): - return _CallableAdapter( - partial(target, **target_params), request.connection - ) - - return None - - -class ReversibleRuleRouter(ReversibleRouter, RuleRouter): - """A rule-based router that implements ``reverse_url`` method. - - Each rule added to this router may have a ``name`` attribute that can be - used to reconstruct an original uri. The actual reconstruction takes place - in a rule's matcher (see `Matcher.reverse`). - """ - - def __init__(self, rules=None): - self.named_rules = {} # type: typing.Dict[str] - super(ReversibleRuleRouter, self).__init__(rules) - - def process_rule(self, rule): - rule = super(ReversibleRuleRouter, self).process_rule(rule) - - if rule.name: - if rule.name in self.named_rules: - app_log.warning( - "Multiple handlers named %s; replacing previous value", - rule.name) - self.named_rules[rule.name] = rule - - return rule - - def reverse_url(self, name, *args): - if name in self.named_rules: - return self.named_rules[name].matcher.reverse(*args) - - for rule in self.rules: - if isinstance(rule.target, ReversibleRouter): - reversed_url = rule.target.reverse_url(name, *args) - if reversed_url is not None: - return reversed_url - - return None - - -class Rule(object): - """A routing rule.""" - - def __init__(self, matcher, target, target_kwargs=None, name=None): - """Constructs a Rule instance. - - :arg Matcher matcher: a `Matcher` instance used for determining - whether the rule should be considered a match for a specific - request. - :arg target: a Rule's target (typically a ``RequestHandler`` or - `~.httputil.HTTPServerConnectionDelegate` subclass or even a nested `Router`, - depending on routing implementation). - :arg dict target_kwargs: a dict of parameters that can be useful - at the moment of target instantiation (for example, ``status_code`` - for a ``RequestHandler`` subclass). They end up in - ``target_params['target_kwargs']`` of `RuleRouter.get_target_delegate` - method. - :arg str name: the name of the rule that can be used to find it - in `ReversibleRouter.reverse_url` implementation. - """ - if isinstance(target, str): - # import the Module and instantiate the class - # Must be a fully qualified name (module.ClassName) - target = import_object(target) - - self.matcher = matcher # type: Matcher - self.target = target - self.target_kwargs = target_kwargs if target_kwargs else {} - self.name = name - - def reverse(self, *args): - return self.matcher.reverse(*args) - - def __repr__(self): - return '%s(%r, %s, kwargs=%r, name=%r)' % \ - (self.__class__.__name__, self.matcher, - self.target, self.target_kwargs, self.name) - - -class Matcher(object): - """Represents a matcher for request features.""" - - def match(self, request): - """Matches current instance against the request. - - :arg httputil.HTTPServerRequest request: current HTTP request - :returns: a dict of parameters to be passed to the target handler - (for example, ``handler_kwargs``, ``path_args``, ``path_kwargs`` - can be passed for proper `~.web.RequestHandler` instantiation). - An empty dict is a valid (and common) return value to indicate a match - when the argument-passing features are not used. - ``None`` must be returned to indicate that there is no match.""" - raise NotImplementedError() - - def reverse(self, *args): - """Reconstructs full url from matcher instance and additional arguments.""" - return None - - -class AnyMatches(Matcher): - """Matches any request.""" - - def match(self, request): - return {} - - -class HostMatches(Matcher): - """Matches requests from hosts specified by ``host_pattern`` regex.""" - - def __init__(self, host_pattern): - if isinstance(host_pattern, basestring_type): - if not host_pattern.endswith("$"): - host_pattern += "$" - self.host_pattern = re.compile(host_pattern) - else: - self.host_pattern = host_pattern - - def match(self, request): - if self.host_pattern.match(request.host_name): - return {} - - return None - - -class DefaultHostMatches(Matcher): - """Matches requests from host that is equal to application's default_host. - Always returns no match if ``X-Real-Ip`` header is present. - """ - - def __init__(self, application, host_pattern): - self.application = application - self.host_pattern = host_pattern - - def match(self, request): - # Look for default host if not behind load balancer (for debugging) - if "X-Real-Ip" not in request.headers: - if self.host_pattern.match(self.application.default_host): - return {} - return None - - -class PathMatches(Matcher): - """Matches requests with paths specified by ``path_pattern`` regex.""" - - def __init__(self, path_pattern): - if isinstance(path_pattern, basestring_type): - if not path_pattern.endswith('$'): - path_pattern += '$' - self.regex = re.compile(path_pattern) - else: - self.regex = path_pattern - - assert len(self.regex.groupindex) in (0, self.regex.groups), \ - ("groups in url regexes must either be all named or all " - "positional: %r" % self.regex.pattern) - - self._path, self._group_count = self._find_groups() - - def match(self, request): - match = self.regex.match(request.path) - if match is None: - return None - if not self.regex.groups: - return {} - - path_args, path_kwargs = [], {} - - # Pass matched groups to the handler. Since - # match.groups() includes both named and - # unnamed groups, we want to use either groups - # or groupdict but not both. - if self.regex.groupindex: - path_kwargs = dict( - (str(k), _unquote_or_none(v)) - for (k, v) in match.groupdict().items()) - else: - path_args = [_unquote_or_none(s) for s in match.groups()] - - return dict(path_args=path_args, path_kwargs=path_kwargs) - - def reverse(self, *args): - if self._path is None: - raise ValueError("Cannot reverse url regex " + self.regex.pattern) - assert len(args) == self._group_count, "required number of arguments " \ - "not found" - if not len(args): - return self._path - converted_args = [] - for a in args: - if not isinstance(a, (unicode_type, bytes)): - a = str(a) - converted_args.append(url_escape(utf8(a), plus=False)) - return self._path % tuple(converted_args) - - def _find_groups(self): - """Returns a tuple (reverse string, group count) for a url. - - For example: Given the url pattern /([0-9]{4})/([a-z-]+)/, this method - would return ('/%s/%s/', 2). - """ - pattern = self.regex.pattern - if pattern.startswith('^'): - pattern = pattern[1:] - if pattern.endswith('$'): - pattern = pattern[:-1] - - if self.regex.groups != pattern.count('('): - # The pattern is too complicated for our simplistic matching, - # so we can't support reversing it. - return None, None - - pieces = [] - for fragment in pattern.split('('): - if ')' in fragment: - paren_loc = fragment.index(')') - if paren_loc >= 0: - pieces.append('%s' + fragment[paren_loc + 1:]) - else: - try: - unescaped_fragment = re_unescape(fragment) - except ValueError: - # If we can't unescape part of it, we can't - # reverse this url. - return (None, None) - pieces.append(unescaped_fragment) - - return ''.join(pieces), self.regex.groups - - -class URLSpec(Rule): - """Specifies mappings between URLs and handlers. - - .. versionchanged: 4.5 - `URLSpec` is now a subclass of a `Rule` with `PathMatches` matcher and is preserved for - backwards compatibility. - """ - def __init__(self, pattern, handler, kwargs=None, name=None): - """Parameters: - - * ``pattern``: Regular expression to be matched. Any capturing - groups in the regex will be passed in to the handler's - get/post/etc methods as arguments (by keyword if named, by - position if unnamed. Named and unnamed capturing groups - may not be mixed in the same rule). - - * ``handler``: `~.web.RequestHandler` subclass to be invoked. - - * ``kwargs`` (optional): A dictionary of additional arguments - to be passed to the handler's constructor. - - * ``name`` (optional): A name for this handler. Used by - `~.web.Application.reverse_url`. - - """ - super(URLSpec, self).__init__(PathMatches(pattern), handler, kwargs, name) - - self.regex = self.matcher.regex - self.handler_class = self.target - self.kwargs = kwargs - - def __repr__(self): - return '%s(%r, %s, kwargs=%r, name=%r)' % \ - (self.__class__.__name__, self.regex.pattern, - self.handler_class, self.kwargs, self.name) - - -def _unquote_or_none(s): - """None-safe wrapper around url_unescape to handle unmatched optional - groups correctly. - - Note that args are passed as bytes so the handler can decide what - encoding to use. - """ - if s is None: - return s - return url_unescape(s, encoding=None, plus=False) diff --git a/lib/tornado/simple_httpclient.py b/lib/tornado/simple_httpclient.py deleted file mode 100755 index 60b7956f..00000000 --- a/lib/tornado/simple_httpclient.py +++ /dev/null @@ -1,566 +0,0 @@ -from __future__ import absolute_import, division, print_function - -from tornado.escape import _unicode -from tornado import gen -from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy -from tornado import httputil -from tornado.http1connection import HTTP1Connection, HTTP1ConnectionParameters -from tornado.ioloop import IOLoop -from tornado.iostream import StreamClosedError -from tornado.netutil import Resolver, OverrideResolver, _client_ssl_defaults -from tornado.log import gen_log -from tornado import stack_context -from tornado.tcpclient import TCPClient -from tornado.util import PY3 - -import base64 -import collections -import copy -import functools -import re -import socket -import sys -import time -from io import BytesIO - - -if PY3: - import urllib.parse as urlparse -else: - import urlparse - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine. - ssl = None - - -class HTTPTimeoutError(HTTPError): - """Error raised by SimpleAsyncHTTPClient on timeout. - - For historical reasons, this is a subclass of `.HTTPClientError` - which simulates a response code of 599. - - .. versionadded:: 5.1 - """ - def __init__(self, message): - super(HTTPTimeoutError, self).__init__(599, message=message) - - def __str__(self): - return self.message - - -class HTTPStreamClosedError(HTTPError): - """Error raised by SimpleAsyncHTTPClient when the underlying stream is closed. - - When a more specific exception is available (such as `ConnectionResetError`), - it may be raised instead of this one. - - For historical reasons, this is a subclass of `.HTTPClientError` - which simulates a response code of 599. - - .. versionadded:: 5.1 - """ - def __init__(self, message): - super(HTTPStreamClosedError, self).__init__(599, message=message) - - def __str__(self): - return self.message - - -class SimpleAsyncHTTPClient(AsyncHTTPClient): - """Non-blocking HTTP client with no external dependencies. - - This class implements an HTTP 1.1 client on top of Tornado's IOStreams. - Some features found in the curl-based AsyncHTTPClient are not yet - supported. In particular, proxies are not supported, connections - are not reused, and callers cannot select the network interface to be - used. - """ - def initialize(self, max_clients=10, - hostname_mapping=None, max_buffer_size=104857600, - resolver=None, defaults=None, max_header_size=None, - max_body_size=None): - """Creates a AsyncHTTPClient. - - Only a single AsyncHTTPClient instance exists per IOLoop - in order to provide limitations on the number of pending connections. - ``force_instance=True`` may be used to suppress this behavior. - - Note that because of this implicit reuse, unless ``force_instance`` - is used, only the first call to the constructor actually uses - its arguments. It is recommended to use the ``configure`` method - instead of the constructor to ensure that arguments take effect. - - ``max_clients`` is the number of concurrent requests that can be - in progress; when this limit is reached additional requests will be - queued. Note that time spent waiting in this queue still counts - against the ``request_timeout``. - - ``hostname_mapping`` is a dictionary mapping hostnames to IP addresses. - It can be used to make local DNS changes when modifying system-wide - settings like ``/etc/hosts`` is not possible or desirable (e.g. in - unittests). - - ``max_buffer_size`` (default 100MB) is the number of bytes - that can be read into memory at once. ``max_body_size`` - (defaults to ``max_buffer_size``) is the largest response body - that the client will accept. Without a - ``streaming_callback``, the smaller of these two limits - applies; with a ``streaming_callback`` only ``max_body_size`` - does. - - .. versionchanged:: 4.2 - Added the ``max_body_size`` argument. - """ - super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults) - self.max_clients = max_clients - self.queue = collections.deque() - self.active = {} - self.waiting = {} - self.max_buffer_size = max_buffer_size - self.max_header_size = max_header_size - self.max_body_size = max_body_size - # TCPClient could create a Resolver for us, but we have to do it - # ourselves to support hostname_mapping. - if resolver: - self.resolver = resolver - self.own_resolver = False - else: - self.resolver = Resolver() - self.own_resolver = True - if hostname_mapping is not None: - self.resolver = OverrideResolver(resolver=self.resolver, - mapping=hostname_mapping) - self.tcp_client = TCPClient(resolver=self.resolver) - - def close(self): - super(SimpleAsyncHTTPClient, self).close() - if self.own_resolver: - self.resolver.close() - self.tcp_client.close() - - def fetch_impl(self, request, callback): - key = object() - self.queue.append((key, request, callback)) - if not len(self.active) < self.max_clients: - timeout_handle = self.io_loop.add_timeout( - self.io_loop.time() + min(request.connect_timeout, - request.request_timeout), - functools.partial(self._on_timeout, key, "in request queue")) - else: - timeout_handle = None - self.waiting[key] = (request, callback, timeout_handle) - self._process_queue() - if self.queue: - gen_log.debug("max_clients limit reached, request queued. " - "%d active, %d queued requests." % ( - len(self.active), len(self.queue))) - - def _process_queue(self): - with stack_context.NullContext(): - while self.queue and len(self.active) < self.max_clients: - key, request, callback = self.queue.popleft() - if key not in self.waiting: - continue - self._remove_timeout(key) - self.active[key] = (request, callback) - release_callback = functools.partial(self._release_fetch, key) - self._handle_request(request, release_callback, callback) - - def _connection_class(self): - return _HTTPConnection - - def _handle_request(self, request, release_callback, final_callback): - self._connection_class()( - self, request, release_callback, - final_callback, self.max_buffer_size, self.tcp_client, - self.max_header_size, self.max_body_size) - - def _release_fetch(self, key): - del self.active[key] - self._process_queue() - - def _remove_timeout(self, key): - if key in self.waiting: - request, callback, timeout_handle = self.waiting[key] - if timeout_handle is not None: - self.io_loop.remove_timeout(timeout_handle) - del self.waiting[key] - - def _on_timeout(self, key, info=None): - """Timeout callback of request. - - Construct a timeout HTTPResponse when a timeout occurs. - - :arg object key: A simple object to mark the request. - :info string key: More detailed timeout information. - """ - request, callback, timeout_handle = self.waiting[key] - self.queue.remove((key, request, callback)) - - error_message = "Timeout {0}".format(info) if info else "Timeout" - timeout_response = HTTPResponse( - request, 599, error=HTTPTimeoutError(error_message), - request_time=self.io_loop.time() - request.start_time) - self.io_loop.add_callback(callback, timeout_response) - del self.waiting[key] - - -class _HTTPConnection(httputil.HTTPMessageDelegate): - _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) - - def __init__(self, client, request, release_callback, - final_callback, max_buffer_size, tcp_client, - max_header_size, max_body_size): - self.io_loop = IOLoop.current() - self.start_time = self.io_loop.time() - self.start_wall_time = time.time() - self.client = client - self.request = request - self.release_callback = release_callback - self.final_callback = final_callback - self.max_buffer_size = max_buffer_size - self.tcp_client = tcp_client - self.max_header_size = max_header_size - self.max_body_size = max_body_size - self.code = None - self.headers = None - self.chunks = [] - self._decompressor = None - # Timeout handle returned by IOLoop.add_timeout - self._timeout = None - self._sockaddr = None - IOLoop.current().add_callback(self.run) - - @gen.coroutine - def run(self): - try: - self.parsed = urlparse.urlsplit(_unicode(self.request.url)) - if self.parsed.scheme not in ("http", "https"): - raise ValueError("Unsupported url scheme: %s" % - self.request.url) - # urlsplit results have hostname and port results, but they - # didn't support ipv6 literals until python 2.7. - netloc = self.parsed.netloc - if "@" in netloc: - userpass, _, netloc = netloc.rpartition("@") - host, port = httputil.split_host_and_port(netloc) - if port is None: - port = 443 if self.parsed.scheme == "https" else 80 - if re.match(r'^\[.*\]$', host): - # raw ipv6 addresses in urls are enclosed in brackets - host = host[1:-1] - self.parsed_hostname = host # save final host for _on_connect - - if self.request.allow_ipv6 is False: - af = socket.AF_INET - else: - af = socket.AF_UNSPEC - - ssl_options = self._get_ssl_options(self.parsed.scheme) - - timeout = min(self.request.connect_timeout, self.request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - stack_context.wrap(functools.partial(self._on_timeout, "while connecting"))) - stream = yield self.tcp_client.connect( - host, port, af=af, - ssl_options=ssl_options, - max_buffer_size=self.max_buffer_size) - - if self.final_callback is None: - # final_callback is cleared if we've hit our timeout. - stream.close() - return - self.stream = stream - self.stream.set_close_callback(self.on_connection_close) - self._remove_timeout() - if self.final_callback is None: - return - if self.request.request_timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + self.request.request_timeout, - stack_context.wrap(functools.partial(self._on_timeout, "during request"))) - if (self.request.method not in self._SUPPORTED_METHODS and - not self.request.allow_nonstandard_methods): - raise KeyError("unknown method %s" % self.request.method) - for key in ('network_interface', - 'proxy_host', 'proxy_port', - 'proxy_username', 'proxy_password', - 'proxy_auth_mode'): - if getattr(self.request, key, None): - raise NotImplementedError('%s not supported' % key) - if "Connection" not in self.request.headers: - self.request.headers["Connection"] = "close" - if "Host" not in self.request.headers: - if '@' in self.parsed.netloc: - self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1] - else: - self.request.headers["Host"] = self.parsed.netloc - username, password = None, None - if self.parsed.username is not None: - username, password = self.parsed.username, self.parsed.password - elif self.request.auth_username is not None: - username = self.request.auth_username - password = self.request.auth_password or '' - if username is not None: - if self.request.auth_mode not in (None, "basic"): - raise ValueError("unsupported auth_mode %s", - self.request.auth_mode) - self.request.headers["Authorization"] = ( - b"Basic " + base64.b64encode( - httputil.encode_username_password(username, password))) - if self.request.user_agent: - self.request.headers["User-Agent"] = self.request.user_agent - if not self.request.allow_nonstandard_methods: - # Some HTTP methods nearly always have bodies while others - # almost never do. Fail in this case unless the user has - # opted out of sanity checks with allow_nonstandard_methods. - body_expected = self.request.method in ("POST", "PATCH", "PUT") - body_present = (self.request.body is not None or - self.request.body_producer is not None) - if ((body_expected and not body_present) or - (body_present and not body_expected)): - raise ValueError( - 'Body must %sbe None for method %s (unless ' - 'allow_nonstandard_methods is true)' % - ('not ' if body_expected else '', self.request.method)) - if self.request.expect_100_continue: - self.request.headers["Expect"] = "100-continue" - if self.request.body is not None: - # When body_producer is used the caller is responsible for - # setting Content-Length (or else chunked encoding will be used). - self.request.headers["Content-Length"] = str(len( - self.request.body)) - if (self.request.method == "POST" and - "Content-Type" not in self.request.headers): - self.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - if self.request.decompress_response: - self.request.headers["Accept-Encoding"] = "gzip" - req_path = ((self.parsed.path or '/') + - (('?' + self.parsed.query) if self.parsed.query else '')) - self.connection = self._create_connection(stream) - start_line = httputil.RequestStartLine(self.request.method, - req_path, '') - self.connection.write_headers(start_line, self.request.headers) - if self.request.expect_100_continue: - yield self.connection.read_response(self) - else: - yield self._write_body(True) - except Exception: - if not self._handle_exception(*sys.exc_info()): - raise - - def _get_ssl_options(self, scheme): - if scheme == "https": - if self.request.ssl_options is not None: - return self.request.ssl_options - # If we are using the defaults, don't construct a - # new SSLContext. - if (self.request.validate_cert and - self.request.ca_certs is None and - self.request.client_cert is None and - self.request.client_key is None): - return _client_ssl_defaults - ssl_ctx = ssl.create_default_context( - ssl.Purpose.SERVER_AUTH, - cafile=self.request.ca_certs) - if not self.request.validate_cert: - ssl_ctx.check_hostname = False - ssl_ctx.verify_mode = ssl.CERT_NONE - if self.request.client_cert is not None: - ssl_ctx.load_cert_chain(self.request.client_cert, - self.request.client_key) - if hasattr(ssl, 'OP_NO_COMPRESSION'): - # See netutil.ssl_options_to_context - ssl_ctx.options |= ssl.OP_NO_COMPRESSION - return ssl_ctx - return None - - def _on_timeout(self, info=None): - """Timeout callback of _HTTPConnection instance. - - Raise a `HTTPTimeoutError` when a timeout occurs. - - :info string key: More detailed timeout information. - """ - self._timeout = None - error_message = "Timeout {0}".format(info) if info else "Timeout" - if self.final_callback is not None: - self._handle_exception(HTTPTimeoutError, HTTPTimeoutError(error_message), - None) - - def _remove_timeout(self): - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = None - - def _create_connection(self, stream): - stream.set_nodelay(True) - connection = HTTP1Connection( - stream, True, - HTTP1ConnectionParameters( - no_keep_alive=True, - max_header_size=self.max_header_size, - max_body_size=self.max_body_size, - decompress=self.request.decompress_response), - self._sockaddr) - return connection - - @gen.coroutine - def _write_body(self, start_read): - if self.request.body is not None: - self.connection.write(self.request.body) - elif self.request.body_producer is not None: - fut = self.request.body_producer(self.connection.write) - if fut is not None: - yield fut - self.connection.finish() - if start_read: - try: - yield self.connection.read_response(self) - except StreamClosedError: - if not self._handle_exception(*sys.exc_info()): - raise - - def _release(self): - if self.release_callback is not None: - release_callback = self.release_callback - self.release_callback = None - release_callback() - - def _run_callback(self, response): - self._release() - if self.final_callback is not None: - final_callback = self.final_callback - self.final_callback = None - self.io_loop.add_callback(final_callback, response) - - def _handle_exception(self, typ, value, tb): - if self.final_callback: - self._remove_timeout() - if isinstance(value, StreamClosedError): - if value.real_error is None: - value = HTTPStreamClosedError("Stream closed") - else: - value = value.real_error - self._run_callback(HTTPResponse(self.request, 599, error=value, - request_time=self.io_loop.time() - self.start_time, - start_time=self.start_wall_time, - )) - - if hasattr(self, "stream"): - # TODO: this may cause a StreamClosedError to be raised - # by the connection's Future. Should we cancel the - # connection more gracefully? - self.stream.close() - return True - else: - # If our callback has already been called, we are probably - # catching an exception that is not caused by us but rather - # some child of our callback. Rather than drop it on the floor, - # pass it along, unless it's just the stream being closed. - return isinstance(value, StreamClosedError) - - def on_connection_close(self): - if self.final_callback is not None: - message = "Connection closed" - if self.stream.error: - raise self.stream.error - try: - raise HTTPStreamClosedError(message) - except HTTPStreamClosedError: - self._handle_exception(*sys.exc_info()) - - def headers_received(self, first_line, headers): - if self.request.expect_100_continue and first_line.code == 100: - self._write_body(False) - return - self.code = first_line.code - self.reason = first_line.reason - self.headers = headers - - if self._should_follow_redirect(): - return - - if self.request.header_callback is not None: - # Reassemble the start line. - self.request.header_callback('%s %s %s\r\n' % first_line) - for k, v in self.headers.get_all(): - self.request.header_callback("%s: %s\r\n" % (k, v)) - self.request.header_callback('\r\n') - - def _should_follow_redirect(self): - return (self.request.follow_redirects and - self.request.max_redirects > 0 and - self.code in (301, 302, 303, 307, 308)) - - def finish(self): - data = b''.join(self.chunks) - self._remove_timeout() - original_request = getattr(self.request, "original_request", - self.request) - if self._should_follow_redirect(): - assert isinstance(self.request, _RequestProxy) - new_request = copy.copy(self.request.request) - new_request.url = urlparse.urljoin(self.request.url, - self.headers["Location"]) - new_request.max_redirects = self.request.max_redirects - 1 - del new_request.headers["Host"] - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - # Client SHOULD make a GET request after a 303. - # According to the spec, 302 should be followed by the same - # method as the original request, but in practice browsers - # treat 302 the same as 303, and many servers use 302 for - # compatibility with pre-HTTP/1.1 user agents which don't - # understand the 303 status. - if self.code in (302, 303): - new_request.method = "GET" - new_request.body = None - for h in ["Content-Length", "Content-Type", - "Content-Encoding", "Transfer-Encoding"]: - try: - del self.request.headers[h] - except KeyError: - pass - new_request.original_request = original_request - final_callback = self.final_callback - self.final_callback = None - self._release() - fut = self.client.fetch(new_request, raise_error=False) - fut.add_done_callback(lambda f: final_callback(f.result())) - self._on_end_request() - return - if self.request.streaming_callback: - buffer = BytesIO() - else: - buffer = BytesIO(data) # TODO: don't require one big string? - response = HTTPResponse(original_request, - self.code, reason=getattr(self, 'reason', None), - headers=self.headers, - request_time=self.io_loop.time() - self.start_time, - start_time=self.start_wall_time, - buffer=buffer, - effective_url=self.request.url) - self._run_callback(response) - self._on_end_request() - - def _on_end_request(self): - self.stream.close() - - def data_received(self, chunk): - if self._should_follow_redirect(): - # We're going to follow a redirect so just discard the body. - return - if self.request.streaming_callback is not None: - self.request.streaming_callback(chunk) - else: - self.chunks.append(chunk) - - -if __name__ == "__main__": - AsyncHTTPClient.configure(SimpleAsyncHTTPClient) - main() diff --git a/lib/tornado/speedups.c b/lib/tornado/speedups.c deleted file mode 100755 index b714268a..00000000 --- a/lib/tornado/speedups.c +++ /dev/null @@ -1,77 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include <Python.h> -#include <stdint.h> - -static PyObject* websocket_mask(PyObject* self, PyObject* args) { - const char* mask; - Py_ssize_t mask_len; - uint32_t uint32_mask; - uint64_t uint64_mask; - const char* data; - Py_ssize_t data_len; - Py_ssize_t i; - PyObject* result; - char* buf; - - if (!PyArg_ParseTuple(args, "s#s#", &mask, &mask_len, &data, &data_len)) { - return NULL; - } - - uint32_mask = ((uint32_t*)mask)[0]; - - result = PyBytes_FromStringAndSize(NULL, data_len); - if (!result) { - return NULL; - } - buf = PyBytes_AsString(result); - - if (sizeof(size_t) >= 8) { - uint64_mask = uint32_mask; - uint64_mask = (uint64_mask << 32) | uint32_mask; - - while (data_len >= 8) { - ((uint64_t*)buf)[0] = ((uint64_t*)data)[0] ^ uint64_mask; - data += 8; - buf += 8; - data_len -= 8; - } - } - - while (data_len >= 4) { - ((uint32_t*)buf)[0] = ((uint32_t*)data)[0] ^ uint32_mask; - data += 4; - buf += 4; - data_len -= 4; - } - - for (i = 0; i < data_len; i++) { - buf[i] = data[i] ^ mask[i]; - } - - return result; -} - -static PyMethodDef methods[] = { - {"websocket_mask", websocket_mask, METH_VARARGS, ""}, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef speedupsmodule = { - PyModuleDef_HEAD_INIT, - "speedups", - NULL, - -1, - methods -}; - -PyMODINIT_FUNC -PyInit_speedups(void) { - return PyModule_Create(&speedupsmodule); -} -#else // Python 2.x -PyMODINIT_FUNC -initspeedups(void) { - Py_InitModule("tornado.speedups", methods); -} -#endif diff --git a/lib/tornado/speedups.pyi b/lib/tornado/speedups.pyi deleted file mode 100755 index 9e8def48..00000000 --- a/lib/tornado/speedups.pyi +++ /dev/null @@ -1 +0,0 @@ -def websocket_mask(mask: bytes, data: bytes) -> bytes: ... diff --git a/lib/tornado/stack_context.py b/lib/tornado/stack_context.py deleted file mode 100755 index a1eca4c7..00000000 --- a/lib/tornado/stack_context.py +++ /dev/null @@ -1,413 +0,0 @@ -# -# Copyright 2010 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""`StackContext` allows applications to maintain threadlocal-like state -that follows execution as it moves to other execution contexts. - -The motivating examples are to eliminate the need for explicit -``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to -allow some additional context to be kept for logging. - -This is slightly magic, but it's an extension of the idea that an -exception handler is a kind of stack-local state and when that stack -is suspended and resumed in a new context that state needs to be -preserved. `StackContext` shifts the burden of restoring that state -from each call site (e.g. wrapping each `.AsyncHTTPClient` callback -in ``async_callback``) to the mechanisms that transfer control from -one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`, -thread pools, etc). - -Example usage:: - - @contextlib.contextmanager - def die_on_error(): - try: - yield - except Exception: - logging.error("exception in asynchronous operation",exc_info=True) - sys.exit(1) - - with StackContext(die_on_error): - # Any exception thrown here *or in callback and its descendants* - # will cause the process to exit instead of spinning endlessly - # in the ioloop. - http_client.fetch(url, callback) - ioloop.start() - -Most applications shouldn't have to work with `StackContext` directly. -Here are a few rules of thumb for when it's necessary: - -* If you're writing an asynchronous library that doesn't rely on a - stack_context-aware library like `tornado.ioloop` or `tornado.iostream` - (for example, if you're writing a thread pool), use - `.stack_context.wrap()` before any asynchronous operations to capture the - stack context from where the operation was started. - -* If you're writing an asynchronous library that has some shared - resources (such as a connection pool), create those shared resources - within a ``with stack_context.NullContext():`` block. This will prevent - ``StackContexts`` from leaking from one request to another. - -* If you want to write something like an exception handler that will - persist across asynchronous calls, create a new `StackContext` (or - `ExceptionStackContext`), and make your asynchronous calls in a ``with`` - block that references your `StackContext`. - -.. deprecated:: 5.1 - - The ``stack_context`` package is deprecated and will be removed in - Tornado 6.0. -""" - -from __future__ import absolute_import, division, print_function - -import sys -import threading -import warnings - -from tornado.util import raise_exc_info - - -class StackContextInconsistentError(Exception): - pass - - -class _State(threading.local): - def __init__(self): - self.contexts = (tuple(), None) - - -_state = _State() - - -class StackContext(object): - """Establishes the given context as a StackContext that will be transferred. - - Note that the parameter is a callable that returns a context - manager, not the context itself. That is, where for a - non-transferable context manager you would say:: - - with my_context(): - - StackContext takes the function itself rather than its result:: - - with StackContext(my_context): - - The result of ``with StackContext() as cb:`` is a deactivation - callback. Run this callback when the StackContext is no longer - needed to ensure that it is not propagated any further (note that - deactivating a context does not affect any instances of that - context that are currently pending). This is an advanced feature - and not necessary in most applications. - """ - def __init__(self, context_factory): - warnings.warn("StackContext is deprecated and will be removed in Tornado 6.0", - DeprecationWarning) - self.context_factory = context_factory - self.contexts = [] - self.active = True - - def _deactivate(self): - self.active = False - - # StackContext protocol - def enter(self): - context = self.context_factory() - self.contexts.append(context) - context.__enter__() - - def exit(self, type, value, traceback): - context = self.contexts.pop() - context.__exit__(type, value, traceback) - - # Note that some of this code is duplicated in ExceptionStackContext - # below. ExceptionStackContext is more common and doesn't need - # the full generality of this class. - def __enter__(self): - self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts[0] + (self,), self) - _state.contexts = self.new_contexts - - try: - self.enter() - except: - _state.contexts = self.old_contexts - raise - - return self._deactivate - - def __exit__(self, type, value, traceback): - try: - self.exit(type, value, traceback) - finally: - final_contexts = _state.contexts - _state.contexts = self.old_contexts - - # Generator coroutines and with-statements with non-local - # effects interact badly. Check here for signs of - # the stack getting out of sync. - # Note that this check comes after restoring _state.context - # so that if it fails things are left in a (relatively) - # consistent state. - if final_contexts is not self.new_contexts: - raise StackContextInconsistentError( - 'stack_context inconsistency (may be caused by yield ' - 'within a "with StackContext" block)') - - # Break up a reference to itself to allow for faster GC on CPython. - self.new_contexts = None - - -class ExceptionStackContext(object): - """Specialization of StackContext for exception handling. - - The supplied ``exception_handler`` function will be called in the - event of an uncaught exception in this context. The semantics are - similar to a try/finally clause, and intended use cases are to log - an error, close a socket, or similar cleanup actions. The - ``exc_info`` triple ``(type, value, traceback)`` will be passed to the - exception_handler function. - - If the exception handler returns true, the exception will be - consumed and will not be propagated to other exception handlers. - - .. versionadded:: 5.1 - - The ``delay_warning`` argument can be used to delay the emission - of DeprecationWarnings until an exception is caught by the - ``ExceptionStackContext``, which facilitates certain transitional - use cases. - """ - def __init__(self, exception_handler, delay_warning=False): - self.delay_warning = delay_warning - if not self.delay_warning: - warnings.warn( - "StackContext is deprecated and will be removed in Tornado 6.0", - DeprecationWarning) - self.exception_handler = exception_handler - self.active = True - - def _deactivate(self): - self.active = False - - def exit(self, type, value, traceback): - if type is not None: - if self.delay_warning: - warnings.warn( - "StackContext is deprecated and will be removed in Tornado 6.0", - DeprecationWarning) - return self.exception_handler(type, value, traceback) - - def __enter__(self): - self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts[0], self) - _state.contexts = self.new_contexts - - return self._deactivate - - def __exit__(self, type, value, traceback): - try: - if type is not None: - return self.exception_handler(type, value, traceback) - finally: - final_contexts = _state.contexts - _state.contexts = self.old_contexts - - if final_contexts is not self.new_contexts: - raise StackContextInconsistentError( - 'stack_context inconsistency (may be caused by yield ' - 'within a "with StackContext" block)') - - # Break up a reference to itself to allow for faster GC on CPython. - self.new_contexts = None - - -class NullContext(object): - """Resets the `StackContext`. - - Useful when creating a shared resource on demand (e.g. an - `.AsyncHTTPClient`) where the stack that caused the creating is - not relevant to future operations. - """ - def __enter__(self): - self.old_contexts = _state.contexts - _state.contexts = (tuple(), None) - - def __exit__(self, type, value, traceback): - _state.contexts = self.old_contexts - - -def _remove_deactivated(contexts): - """Remove deactivated handlers from the chain""" - # Clean ctx handlers - stack_contexts = tuple([h for h in contexts[0] if h.active]) - - # Find new head - head = contexts[1] - while head is not None and not head.active: - head = head.old_contexts[1] - - # Process chain - ctx = head - while ctx is not None: - parent = ctx.old_contexts[1] - - while parent is not None: - if parent.active: - break - ctx.old_contexts = parent.old_contexts - parent = parent.old_contexts[1] - - ctx = parent - - return (stack_contexts, head) - - -def wrap(fn): - """Returns a callable object that will restore the current `StackContext` - when executed. - - Use this whenever saving a callback to be executed later in a - different execution context (either in a different thread or - asynchronously in the same thread). - """ - # Check if function is already wrapped - if fn is None or hasattr(fn, '_wrapped'): - return fn - - # Capture current stack head - # TODO: Any other better way to store contexts and update them in wrapped function? - cap_contexts = [_state.contexts] - - if not cap_contexts[0][0] and not cap_contexts[0][1]: - # Fast path when there are no active contexts. - def null_wrapper(*args, **kwargs): - try: - current_state = _state.contexts - _state.contexts = cap_contexts[0] - return fn(*args, **kwargs) - finally: - _state.contexts = current_state - null_wrapper._wrapped = True - return null_wrapper - - def wrapped(*args, **kwargs): - ret = None - try: - # Capture old state - current_state = _state.contexts - - # Remove deactivated items - cap_contexts[0] = contexts = _remove_deactivated(cap_contexts[0]) - - # Force new state - _state.contexts = contexts - - # Current exception - exc = (None, None, None) - top = None - - # Apply stack contexts - last_ctx = 0 - stack = contexts[0] - - # Apply state - for n in stack: - try: - n.enter() - last_ctx += 1 - except: - # Exception happened. Record exception info and store top-most handler - exc = sys.exc_info() - top = n.old_contexts[1] - - # Execute callback if no exception happened while restoring state - if top is None: - try: - ret = fn(*args, **kwargs) - except: - exc = sys.exc_info() - top = contexts[1] - - # If there was exception, try to handle it by going through the exception chain - if top is not None: - exc = _handle_exception(top, exc) - else: - # Otherwise take shorter path and run stack contexts in reverse order - while last_ctx > 0: - last_ctx -= 1 - c = stack[last_ctx] - - try: - c.exit(*exc) - except: - exc = sys.exc_info() - top = c.old_contexts[1] - break - else: - top = None - - # If if exception happened while unrolling, take longer exception handler path - if top is not None: - exc = _handle_exception(top, exc) - - # If exception was not handled, raise it - if exc != (None, None, None): - raise_exc_info(exc) - finally: - _state.contexts = current_state - return ret - - wrapped._wrapped = True - return wrapped - - -def _handle_exception(tail, exc): - while tail is not None: - try: - if tail.exit(*exc): - exc = (None, None, None) - except: - exc = sys.exc_info() - - tail = tail.old_contexts[1] - - return exc - - -def run_with_stack_context(context, func): - """Run a coroutine ``func`` in the given `StackContext`. - - It is not safe to have a ``yield`` statement within a ``with StackContext`` - block, so it is difficult to use stack context with `.gen.coroutine`. - This helper function runs the function in the correct context while - keeping the ``yield`` and ``with`` statements syntactically separate. - - Example:: - - @gen.coroutine - def incorrect(): - with StackContext(ctx): - # ERROR: this will raise StackContextInconsistentError - yield other_coroutine() - - @gen.coroutine - def correct(): - yield run_with_stack_context(StackContext(ctx), other_coroutine) - - .. versionadded:: 3.1 - """ - with context: - return func() diff --git a/lib/tornado/tcpclient.py b/lib/tornado/tcpclient.py deleted file mode 100755 index 3a1b58ca..00000000 --- a/lib/tornado/tcpclient.py +++ /dev/null @@ -1,276 +0,0 @@ -# -# Copyright 2014 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A non-blocking TCP connection factory. -""" -from __future__ import absolute_import, division, print_function - -import functools -import socket -import numbers -import datetime - -from tornado.concurrent import Future, future_add_done_callback -from tornado.ioloop import IOLoop -from tornado.iostream import IOStream -from tornado import gen -from tornado.netutil import Resolver -from tornado.platform.auto import set_close_exec -from tornado.gen import TimeoutError -from tornado.util import timedelta_to_seconds - -_INITIAL_CONNECT_TIMEOUT = 0.3 - - -class _Connector(object): - """A stateless implementation of the "Happy Eyeballs" algorithm. - - "Happy Eyeballs" is documented in RFC6555 as the recommended practice - for when both IPv4 and IPv6 addresses are available. - - In this implementation, we partition the addresses by family, and - make the first connection attempt to whichever address was - returned first by ``getaddrinfo``. If that connection fails or - times out, we begin a connection in parallel to the first address - of the other family. If there are additional failures we retry - with other addresses, keeping one connection attempt per family - in flight at a time. - - http://tools.ietf.org/html/rfc6555 - - """ - def __init__(self, addrinfo, connect): - self.io_loop = IOLoop.current() - self.connect = connect - - self.future = Future() - self.timeout = None - self.connect_timeout = None - self.last_error = None - self.remaining = len(addrinfo) - self.primary_addrs, self.secondary_addrs = self.split(addrinfo) - self.streams = set() - - @staticmethod - def split(addrinfo): - """Partition the ``addrinfo`` list by address family. - - Returns two lists. The first list contains the first entry from - ``addrinfo`` and all others with the same family, and the - second list contains all other addresses (normally one list will - be AF_INET and the other AF_INET6, although non-standard resolvers - may return additional families). - """ - primary = [] - secondary = [] - primary_af = addrinfo[0][0] - for af, addr in addrinfo: - if af == primary_af: - primary.append((af, addr)) - else: - secondary.append((af, addr)) - return primary, secondary - - def start(self, timeout=_INITIAL_CONNECT_TIMEOUT, connect_timeout=None): - self.try_connect(iter(self.primary_addrs)) - self.set_timeout(timeout) - if connect_timeout is not None: - self.set_connect_timeout(connect_timeout) - return self.future - - def try_connect(self, addrs): - try: - af, addr = next(addrs) - except StopIteration: - # We've reached the end of our queue, but the other queue - # might still be working. Send a final error on the future - # only when both queues are finished. - if self.remaining == 0 and not self.future.done(): - self.future.set_exception(self.last_error or - IOError("connection failed")) - return - stream, future = self.connect(af, addr) - self.streams.add(stream) - future_add_done_callback( - future, functools.partial(self.on_connect_done, addrs, af, addr)) - - def on_connect_done(self, addrs, af, addr, future): - self.remaining -= 1 - try: - stream = future.result() - except Exception as e: - if self.future.done(): - return - # Error: try again (but remember what happened so we have an - # error to raise in the end) - self.last_error = e - self.try_connect(addrs) - if self.timeout is not None: - # If the first attempt failed, don't wait for the - # timeout to try an address from the secondary queue. - self.io_loop.remove_timeout(self.timeout) - self.on_timeout() - return - self.clear_timeouts() - if self.future.done(): - # This is a late arrival; just drop it. - stream.close() - else: - self.streams.discard(stream) - self.future.set_result((af, addr, stream)) - self.close_streams() - - def set_timeout(self, timeout): - self.timeout = self.io_loop.add_timeout(self.io_loop.time() + timeout, - self.on_timeout) - - def on_timeout(self): - self.timeout = None - if not self.future.done(): - self.try_connect(iter(self.secondary_addrs)) - - def clear_timeout(self): - if self.timeout is not None: - self.io_loop.remove_timeout(self.timeout) - - def set_connect_timeout(self, connect_timeout): - self.connect_timeout = self.io_loop.add_timeout( - connect_timeout, self.on_connect_timeout) - - def on_connect_timeout(self): - if not self.future.done(): - self.future.set_exception(TimeoutError()) - self.close_streams() - - def clear_timeouts(self): - if self.timeout is not None: - self.io_loop.remove_timeout(self.timeout) - if self.connect_timeout is not None: - self.io_loop.remove_timeout(self.connect_timeout) - - def close_streams(self): - for stream in self.streams: - stream.close() - - -class TCPClient(object): - """A non-blocking TCP connection factory. - - .. versionchanged:: 5.0 - The ``io_loop`` argument (deprecated since version 4.1) has been removed. - """ - def __init__(self, resolver=None): - if resolver is not None: - self.resolver = resolver - self._own_resolver = False - else: - self.resolver = Resolver() - self._own_resolver = True - - def close(self): - if self._own_resolver: - self.resolver.close() - - @gen.coroutine - def connect(self, host, port, af=socket.AF_UNSPEC, ssl_options=None, - max_buffer_size=None, source_ip=None, source_port=None, - timeout=None): - """Connect to the given host and port. - - Asynchronously returns an `.IOStream` (or `.SSLIOStream` if - ``ssl_options`` is not None). - - Using the ``source_ip`` kwarg, one can specify the source - IP address to use when establishing the connection. - In case the user needs to resolve and - use a specific interface, it has to be handled outside - of Tornado as this depends very much on the platform. - - Raises `TimeoutError` if the input future does not complete before - ``timeout``, which may be specified in any form allowed by - `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time - relative to `.IOLoop.time`) - - Similarly, when the user requires a certain source port, it can - be specified using the ``source_port`` arg. - - .. versionchanged:: 4.5 - Added the ``source_ip`` and ``source_port`` arguments. - - .. versionchanged:: 5.0 - Added the ``timeout`` argument. - """ - if timeout is not None: - if isinstance(timeout, numbers.Real): - timeout = IOLoop.current().time() + timeout - elif isinstance(timeout, datetime.timedelta): - timeout = IOLoop.current().time() + timedelta_to_seconds(timeout) - else: - raise TypeError("Unsupported timeout %r" % timeout) - if timeout is not None: - addrinfo = yield gen.with_timeout( - timeout, self.resolver.resolve(host, port, af)) - else: - addrinfo = yield self.resolver.resolve(host, port, af) - connector = _Connector( - addrinfo, - functools.partial(self._create_stream, max_buffer_size, - source_ip=source_ip, source_port=source_port) - ) - af, addr, stream = yield connector.start(connect_timeout=timeout) - # TODO: For better performance we could cache the (af, addr) - # information here and re-use it on subsequent connections to - # the same host. (http://tools.ietf.org/html/rfc6555#section-4.2) - if ssl_options is not None: - if timeout is not None: - stream = yield gen.with_timeout(timeout, stream.start_tls( - False, ssl_options=ssl_options, server_hostname=host)) - else: - stream = yield stream.start_tls(False, ssl_options=ssl_options, - server_hostname=host) - raise gen.Return(stream) - - def _create_stream(self, max_buffer_size, af, addr, source_ip=None, - source_port=None): - # Always connect in plaintext; we'll convert to ssl if necessary - # after one connection has completed. - source_port_bind = source_port if isinstance(source_port, int) else 0 - source_ip_bind = source_ip - if source_port_bind and not source_ip: - # User required a specific port, but did not specify - # a certain source IP, will bind to the default loopback. - source_ip_bind = '::1' if af == socket.AF_INET6 else '127.0.0.1' - # Trying to use the same address family as the requested af socket: - # - 127.0.0.1 for IPv4 - # - ::1 for IPv6 - socket_obj = socket.socket(af) - set_close_exec(socket_obj.fileno()) - if source_port_bind or source_ip_bind: - # If the user requires binding also to a specific IP/port. - try: - socket_obj.bind((source_ip_bind, source_port_bind)) - except socket.error: - socket_obj.close() - # Fail loudly if unable to use the IP/port. - raise - try: - stream = IOStream(socket_obj, - max_buffer_size=max_buffer_size) - except socket.error as e: - fu = Future() - fu.set_exception(e) - return fu - else: - return stream, stream.connect(addr) diff --git a/lib/tornado/tcpserver.py b/lib/tornado/tcpserver.py deleted file mode 100755 index 4f5d6f03..00000000 --- a/lib/tornado/tcpserver.py +++ /dev/null @@ -1,299 +0,0 @@ -# -# Copyright 2011 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A non-blocking, single-threaded TCP server.""" -from __future__ import absolute_import, division, print_function - -import errno -import os -import socket - -from tornado import gen -from tornado.log import app_log -from tornado.ioloop import IOLoop -from tornado.iostream import IOStream, SSLIOStream -from tornado.netutil import bind_sockets, add_accept_handler, ssl_wrap_socket -from tornado import process -from tornado.util import errno_from_exception - -try: - import ssl -except ImportError: - # ssl is not available on Google App Engine. - ssl = None - - -class TCPServer(object): - r"""A non-blocking, single-threaded TCP server. - - To use `TCPServer`, define a subclass which overrides the `handle_stream` - method. For example, a simple echo server could be defined like this:: - - from tornado.tcpserver import TCPServer - from tornado.iostream import StreamClosedError - from tornado import gen - - class EchoServer(TCPServer): - async def handle_stream(self, stream, address): - while True: - try: - data = await stream.read_until(b"\n") - await stream.write(data) - except StreamClosedError: - break - - To make this server serve SSL traffic, send the ``ssl_options`` keyword - argument with an `ssl.SSLContext` object. For compatibility with older - versions of Python ``ssl_options`` may also be a dictionary of keyword - arguments for the `ssl.wrap_socket` method.:: - - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain(os.path.join(data_dir, "mydomain.crt"), - os.path.join(data_dir, "mydomain.key")) - TCPServer(ssl_options=ssl_ctx) - - `TCPServer` initialization follows one of three patterns: - - 1. `listen`: simple single-process:: - - server = TCPServer() - server.listen(8888) - IOLoop.current().start() - - 2. `bind`/`start`: simple multi-process:: - - server = TCPServer() - server.bind(8888) - server.start(0) # Forks multiple sub-processes - IOLoop.current().start() - - When using this interface, an `.IOLoop` must *not* be passed - to the `TCPServer` constructor. `start` will always start - the server on the default singleton `.IOLoop`. - - 3. `add_sockets`: advanced multi-process:: - - sockets = bind_sockets(8888) - tornado.process.fork_processes(0) - server = TCPServer() - server.add_sockets(sockets) - IOLoop.current().start() - - The `add_sockets` interface is more complicated, but it can be - used with `tornado.process.fork_processes` to give you more - flexibility in when the fork happens. `add_sockets` can - also be used in single-process servers if you want to create - your listening sockets in some way other than - `~tornado.netutil.bind_sockets`. - - .. versionadded:: 3.1 - The ``max_buffer_size`` argument. - - .. versionchanged:: 5.0 - The ``io_loop`` argument has been removed. - """ - def __init__(self, ssl_options=None, max_buffer_size=None, - read_chunk_size=None): - self.ssl_options = ssl_options - self._sockets = {} # fd -> socket object - self._handlers = {} # fd -> remove_handler callable - self._pending_sockets = [] - self._started = False - self._stopped = False - self.max_buffer_size = max_buffer_size - self.read_chunk_size = read_chunk_size - - # Verify the SSL options. Otherwise we don't get errors until clients - # connect. This doesn't verify that the keys are legitimate, but - # the SSL module doesn't do that until there is a connected socket - # which seems like too much work - if self.ssl_options is not None and isinstance(self.ssl_options, dict): - # Only certfile is required: it can contain both keys - if 'certfile' not in self.ssl_options: - raise KeyError('missing key "certfile" in ssl_options') - - if not os.path.exists(self.ssl_options['certfile']): - raise ValueError('certfile "%s" does not exist' % - self.ssl_options['certfile']) - if ('keyfile' in self.ssl_options and - not os.path.exists(self.ssl_options['keyfile'])): - raise ValueError('keyfile "%s" does not exist' % - self.ssl_options['keyfile']) - - def listen(self, port, address=""): - """Starts accepting connections on the given port. - - This method may be called more than once to listen on multiple ports. - `listen` takes effect immediately; it is not necessary to call - `TCPServer.start` afterwards. It is, however, necessary to start - the `.IOLoop`. - """ - sockets = bind_sockets(port, address=address) - self.add_sockets(sockets) - - def add_sockets(self, sockets): - """Makes this server start accepting connections on the given sockets. - - The ``sockets`` parameter is a list of socket objects such as - those returned by `~tornado.netutil.bind_sockets`. - `add_sockets` is typically used in combination with that - method and `tornado.process.fork_processes` to provide greater - control over the initialization of a multi-process server. - """ - for sock in sockets: - self._sockets[sock.fileno()] = sock - self._handlers[sock.fileno()] = add_accept_handler( - sock, self._handle_connection) - - def add_socket(self, socket): - """Singular version of `add_sockets`. Takes a single socket object.""" - self.add_sockets([socket]) - - def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128, - reuse_port=False): - """Binds this server to the given port on the given address. - - To start the server, call `start`. If you want to run this server - in a single process, you can call `listen` as a shortcut to the - sequence of `bind` and `start` calls. - - Address may be either an IP address or hostname. If it's a hostname, - the server will listen on all IP addresses associated with the - name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either `socket.AF_INET` - or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise - both will be used if available. - - The ``backlog`` argument has the same meaning as for - `socket.listen <socket.socket.listen>`. The ``reuse_port`` argument - has the same meaning as for `.bind_sockets`. - - This method may be called multiple times prior to `start` to listen - on multiple ports or interfaces. - - .. versionchanged:: 4.4 - Added the ``reuse_port`` argument. - """ - sockets = bind_sockets(port, address=address, family=family, - backlog=backlog, reuse_port=reuse_port) - if self._started: - self.add_sockets(sockets) - else: - self._pending_sockets.extend(sockets) - - def start(self, num_processes=1): - """Starts this server in the `.IOLoop`. - - By default, we run the server in this process and do not fork any - additional child process. - - If num_processes is ``None`` or <= 0, we detect the number of cores - available on this machine and fork that number of child - processes. If num_processes is given and > 1, we fork that - specific number of sub-processes. - - Since we use processes and not threads, there is no shared memory - between any server code. - - Note that multiple processes are not compatible with the autoreload - module (or the ``autoreload=True`` option to `tornado.web.Application` - which defaults to True when ``debug=True``). - When using multiple processes, no IOLoops can be created or - referenced until after the call to ``TCPServer.start(n)``. - """ - assert not self._started - self._started = True - if num_processes != 1: - process.fork_processes(num_processes) - sockets = self._pending_sockets - self._pending_sockets = [] - self.add_sockets(sockets) - - def stop(self): - """Stops listening for new connections. - - Requests currently in progress may still continue after the - server is stopped. - """ - if self._stopped: - return - self._stopped = True - for fd, sock in self._sockets.items(): - assert sock.fileno() == fd - # Unregister socket from IOLoop - self._handlers.pop(fd)() - sock.close() - - def handle_stream(self, stream, address): - """Override to handle a new `.IOStream` from an incoming connection. - - This method may be a coroutine; if so any exceptions it raises - asynchronously will be logged. Accepting of incoming connections - will not be blocked by this coroutine. - - If this `TCPServer` is configured for SSL, ``handle_stream`` - may be called before the SSL handshake has completed. Use - `.SSLIOStream.wait_for_handshake` if you need to verify the client's - certificate or use NPN/ALPN. - - .. versionchanged:: 4.2 - Added the option for this method to be a coroutine. - """ - raise NotImplementedError() - - def _handle_connection(self, connection, address): - if self.ssl_options is not None: - assert ssl, "Python 2.6+ and OpenSSL required for SSL" - try: - connection = ssl_wrap_socket(connection, - self.ssl_options, - server_side=True, - do_handshake_on_connect=False) - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_EOF: - return connection.close() - else: - raise - except socket.error as err: - # If the connection is closed immediately after it is created - # (as in a port scan), we can get one of several errors. - # wrap_socket makes an internal call to getpeername, - # which may return either EINVAL (Mac OS X) or ENOTCONN - # (Linux). If it returns ENOTCONN, this error is - # silently swallowed by the ssl module, so we need to - # catch another error later on (AttributeError in - # SSLIOStream._do_ssl_handshake). - # To test this behavior, try nmap with the -sT flag. - # https://github.com/tornadoweb/tornado/pull/750 - if errno_from_exception(err) in (errno.ECONNABORTED, errno.EINVAL): - return connection.close() - else: - raise - try: - if self.ssl_options is not None: - stream = SSLIOStream(connection, - max_buffer_size=self.max_buffer_size, - read_chunk_size=self.read_chunk_size) - else: - stream = IOStream(connection, - max_buffer_size=self.max_buffer_size, - read_chunk_size=self.read_chunk_size) - - future = self.handle_stream(stream, address) - if future is not None: - IOLoop.current().add_future(gen.convert_yielded(future), - lambda f: f.result()) - except Exception: - app_log.error("Error in connection callback", exc_info=True) diff --git a/lib/tornado/template.py b/lib/tornado/template.py deleted file mode 100755 index 61b98746..00000000 --- a/lib/tornado/template.py +++ /dev/null @@ -1,976 +0,0 @@ -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A simple template system that compiles templates to Python code. - -Basic usage looks like:: - - t = template.Template("<html>{{ myvalue }}</html>") - print(t.generate(myvalue="XXX")) - -`Loader` is a class that loads templates from a root directory and caches -the compiled templates:: - - loader = template.Loader("/home/btaylor") - print(loader.load("test.html").generate(myvalue="XXX")) - -We compile all templates to raw Python. Error-reporting is currently... uh, -interesting. Syntax for the templates:: - - ### base.html - <html> - <head> - <title>{% block title %}Default title{% end %} - - - - - - - ### bold.html - {% extends "base.html" %} - - {% block title %}A bolder title{% end %} - - {% block student %} -
  • {{ escape(student.name) }}
    • - {% end %} - -Unlike most other template systems, we do not put any restrictions on the -expressions you can include in your statements. ``if`` and ``for`` blocks get -translated exactly into Python, so you can do complex expressions like:: - - {% for student in [p for p in people if p.student and p.age > 23] %} -
  • {{ escape(student.name) }}
    • - {% end %} - -Translating directly to Python means you can apply functions to expressions -easily, like the ``escape()`` function in the examples above. You can pass -functions in to your template just like any other variable -(In a `.RequestHandler`, override `.RequestHandler.get_template_namespace`):: - - ### Python code - def add(x, y): - return x + y - template.execute(add=add) - - ### The template - {{ add(1, 2) }} - -We provide the functions `escape() <.xhtml_escape>`, `.url_escape()`, -`.json_encode()`, and `.squeeze()` to all templates by default. - -Typical applications do not create `Template` or `Loader` instances by -hand, but instead use the `~.RequestHandler.render` and -`~.RequestHandler.render_string` methods of -`tornado.web.RequestHandler`, which load templates automatically based -on the ``template_path`` `.Application` setting. - -Variable names beginning with ``_tt_`` are reserved by the template -system and should not be used by application code. - -Syntax Reference ----------------- - -Template expressions are surrounded by double curly braces: ``{{ ... }}``. -The contents may be any python expression, which will be escaped according -to the current autoescape setting and inserted into the output. Other -template directives use ``{% %}``. - -To comment out a section so that it is omitted from the output, surround it -with ``{# ... #}``. - -These tags may be escaped as ``{{!``, ``{%!``, and ``{#!`` -if you need to include a literal ``{{``, ``{%``, or ``{#`` in the output. - - -``{% apply *function* %}...{% end %}`` - Applies a function to the output of all template code between ``apply`` - and ``end``:: - - {% apply linkify %}{{name}} said: {{message}}{% end %} - - Note that as an implementation detail apply blocks are implemented - as nested functions and thus may interact strangely with variables - set via ``{% set %}``, or the use of ``{% break %}`` or ``{% continue %}`` - within loops. - -``{% autoescape *function* %}`` - Sets the autoescape mode for the current file. This does not affect - other files, even those referenced by ``{% include %}``. Note that - autoescaping can also be configured globally, at the `.Application` - or `Loader`.:: - - {% autoescape xhtml_escape %} - {% autoescape None %} - -``{% block *name* %}...{% end %}`` - Indicates a named, replaceable block for use with ``{% extends %}``. - Blocks in the parent template will be replaced with the contents of - the same-named block in a child template.:: - - - {% block title %}Default title{% end %} - - - {% extends "base.html" %} - {% block title %}My page title{% end %} - -``{% comment ... %}`` - A comment which will be removed from the template output. Note that - there is no ``{% end %}`` tag; the comment goes from the word ``comment`` - to the closing ``%}`` tag. - -``{% extends *filename* %}`` - Inherit from another template. Templates that use ``extends`` should - contain one or more ``block`` tags to replace content from the parent - template. Anything in the child template not contained in a ``block`` - tag will be ignored. For an example, see the ``{% block %}`` tag. - -``{% for *var* in *expr* %}...{% end %}`` - Same as the python ``for`` statement. ``{% break %}`` and - ``{% continue %}`` may be used inside the loop. - -``{% from *x* import *y* %}`` - Same as the python ``import`` statement. - -``{% if *condition* %}...{% elif *condition* %}...{% else %}...{% end %}`` - Conditional statement - outputs the first section whose condition is - true. (The ``elif`` and ``else`` sections are optional) - -``{% import *module* %}`` - Same as the python ``import`` statement. - -``{% include *filename* %}`` - Includes another template file. The included file can see all the local - variables as if it were copied directly to the point of the ``include`` - directive (the ``{% autoescape %}`` directive is an exception). - Alternately, ``{% module Template(filename, **kwargs) %}`` may be used - to include another template with an isolated namespace. - -``{% module *expr* %}`` - Renders a `~tornado.web.UIModule`. The output of the ``UIModule`` is - not escaped:: - - {% module Template("foo.html", arg=42) %} - - ``UIModules`` are a feature of the `tornado.web.RequestHandler` - class (and specifically its ``render`` method) and will not work - when the template system is used on its own in other contexts. - -``{% raw *expr* %}`` - Outputs the result of the given expression without autoescaping. - -``{% set *x* = *y* %}`` - Sets a local variable. - -``{% try %}...{% except %}...{% else %}...{% finally %}...{% end %}`` - Same as the python ``try`` statement. - -``{% while *condition* %}... {% end %}`` - Same as the python ``while`` statement. ``{% break %}`` and - ``{% continue %}`` may be used inside the loop. - -``{% whitespace *mode* %}`` - Sets the whitespace mode for the remainder of the current file - (or until the next ``{% whitespace %}`` directive). See - `filter_whitespace` for available options. New in Tornado 4.3. -""" - -from __future__ import absolute_import, division, print_function - -import datetime -import linecache -import os.path -import posixpath -import re -import threading - -from tornado import escape -from tornado.log import app_log -from tornado.util import ObjectDict, exec_in, unicode_type, PY3 - -if PY3: - from io import StringIO -else: - from cStringIO import StringIO - -_DEFAULT_AUTOESCAPE = "xhtml_escape" -_UNSET = object() - - -def filter_whitespace(mode, text): - """Transform whitespace in ``text`` according to ``mode``. - - Available modes are: - - * ``all``: Return all whitespace unmodified. - * ``single``: Collapse consecutive whitespace with a single whitespace - character, preserving newlines. - * ``oneline``: Collapse all runs of whitespace into a single space - character, removing all newlines in the process. - - .. versionadded:: 4.3 - """ - if mode == 'all': - return text - elif mode == 'single': - text = re.sub(r"([\t ]+)", " ", text) - text = re.sub(r"(\s*\n\s*)", "\n", text) - return text - elif mode == 'oneline': - return re.sub(r"(\s+)", " ", text) - else: - raise Exception("invalid whitespace mode %s" % mode) - - -class Template(object): - """A compiled template. - - We compile into Python from the given template_string. You can generate - the template from variables with generate(). - """ - # note that the constructor's signature is not extracted with - # autodoc because _UNSET looks like garbage. When changing - # this signature update website/sphinx/template.rst too. - def __init__(self, template_string, name="", loader=None, - compress_whitespace=_UNSET, autoescape=_UNSET, - whitespace=None): - """Construct a Template. - - :arg str template_string: the contents of the template file. - :arg str name: the filename from which the template was loaded - (used for error message). - :arg tornado.template.BaseLoader loader: the `~tornado.template.BaseLoader` responsible - for this template, used to resolve ``{% include %}`` and ``{% extend %}`` directives. - :arg bool compress_whitespace: Deprecated since Tornado 4.3. - Equivalent to ``whitespace="single"`` if true and - ``whitespace="all"`` if false. - :arg str autoescape: The name of a function in the template - namespace, or ``None`` to disable escaping by default. - :arg str whitespace: A string specifying treatment of whitespace; - see `filter_whitespace` for options. - - .. versionchanged:: 4.3 - Added ``whitespace`` parameter; deprecated ``compress_whitespace``. - """ - self.name = escape.native_str(name) - - if compress_whitespace is not _UNSET: - # Convert deprecated compress_whitespace (bool) to whitespace (str). - if whitespace is not None: - raise Exception("cannot set both whitespace and compress_whitespace") - whitespace = "single" if compress_whitespace else "all" - if whitespace is None: - if loader and loader.whitespace: - whitespace = loader.whitespace - else: - # Whitespace defaults by filename. - if name.endswith(".html") or name.endswith(".js"): - whitespace = "single" - else: - whitespace = "all" - # Validate the whitespace setting. - filter_whitespace(whitespace, '') - - if autoescape is not _UNSET: - self.autoescape = autoescape - elif loader: - self.autoescape = loader.autoescape - else: - self.autoescape = _DEFAULT_AUTOESCAPE - - self.namespace = loader.namespace if loader else {} - reader = _TemplateReader(name, escape.native_str(template_string), - whitespace) - self.file = _File(self, _parse(reader, self)) - self.code = self._generate_python(loader) - self.loader = loader - try: - # Under python2.5, the fake filename used here must match - # the module name used in __name__ below. - # The dont_inherit flag prevents template.py's future imports - # from being applied to the generated code. - self.compiled = compile( - escape.to_unicode(self.code), - "%s.generated.py" % self.name.replace('.', '_'), - "exec", dont_inherit=True) - except Exception: - formatted_code = _format_code(self.code).rstrip() - app_log.error("%s code:\n%s", self.name, formatted_code) - raise - - def generate(self, **kwargs): - """Generate this template with the given arguments.""" - namespace = { - "escape": escape.xhtml_escape, - "xhtml_escape": escape.xhtml_escape, - "url_escape": escape.url_escape, - "json_encode": escape.json_encode, - "squeeze": escape.squeeze, - "linkify": escape.linkify, - "datetime": datetime, - "_tt_utf8": escape.utf8, # for internal use - "_tt_string_types": (unicode_type, bytes), - # __name__ and __loader__ allow the traceback mechanism to find - # the generated source code. - "__name__": self.name.replace('.', '_'), - "__loader__": ObjectDict(get_source=lambda name: self.code), - } - namespace.update(self.namespace) - namespace.update(kwargs) - exec_in(self.compiled, namespace) - execute = namespace["_tt_execute"] - # Clear the traceback module's cache of source data now that - # we've generated a new template (mainly for this module's - # unittests, where different tests reuse the same name). - linecache.clearcache() - return execute() - - def _generate_python(self, loader): - buffer = StringIO() - try: - # named_blocks maps from names to _NamedBlock objects - named_blocks = {} - ancestors = self._get_ancestors(loader) - ancestors.reverse() - for ancestor in ancestors: - ancestor.find_named_blocks(loader, named_blocks) - writer = _CodeWriter(buffer, named_blocks, loader, - ancestors[0].template) - ancestors[0].generate(writer) - return buffer.getvalue() - finally: - buffer.close() - - def _get_ancestors(self, loader): - ancestors = [self.file] - for chunk in self.file.body.chunks: - if isinstance(chunk, _ExtendsBlock): - if not loader: - raise ParseError("{% extends %} block found, but no " - "template loader") - template = loader.load(chunk.name, self.name) - ancestors.extend(template._get_ancestors(loader)) - return ancestors - - -class BaseLoader(object): - """Base class for template loaders. - - You must use a template loader to use template constructs like - ``{% extends %}`` and ``{% include %}``. The loader caches all - templates after they are loaded the first time. - """ - def __init__(self, autoescape=_DEFAULT_AUTOESCAPE, namespace=None, - whitespace=None): - """Construct a template loader. - - :arg str autoescape: The name of a function in the template - namespace, such as "xhtml_escape", or ``None`` to disable - autoescaping by default. - :arg dict namespace: A dictionary to be added to the default template - namespace, or ``None``. - :arg str whitespace: A string specifying default behavior for - whitespace in templates; see `filter_whitespace` for options. - Default is "single" for files ending in ".html" and ".js" and - "all" for other files. - - .. versionchanged:: 4.3 - Added ``whitespace`` parameter. - """ - self.autoescape = autoescape - self.namespace = namespace or {} - self.whitespace = whitespace - self.templates = {} - # self.lock protects self.templates. It's a reentrant lock - # because templates may load other templates via `include` or - # `extends`. Note that thanks to the GIL this code would be safe - # even without the lock, but could lead to wasted work as multiple - # threads tried to compile the same template simultaneously. - self.lock = threading.RLock() - - def reset(self): - """Resets the cache of compiled templates.""" - with self.lock: - self.templates = {} - - def resolve_path(self, name, parent_path=None): - """Converts a possibly-relative path to absolute (used internally).""" - raise NotImplementedError() - - def load(self, name, parent_path=None): - """Loads a template.""" - name = self.resolve_path(name, parent_path=parent_path) - with self.lock: - if name not in self.templates: - self.templates[name] = self._create_template(name) - return self.templates[name] - - def _create_template(self, name): - raise NotImplementedError() - - -class Loader(BaseLoader): - """A template loader that loads from a single root directory. - """ - def __init__(self, root_directory, **kwargs): - super(Loader, self).__init__(**kwargs) - self.root = os.path.abspath(root_directory) - - def resolve_path(self, name, parent_path=None): - if parent_path and not parent_path.startswith("<") and \ - not parent_path.startswith("/") and \ - not name.startswith("/"): - current_path = os.path.join(self.root, parent_path) - file_dir = os.path.dirname(os.path.abspath(current_path)) - relative_path = os.path.abspath(os.path.join(file_dir, name)) - if relative_path.startswith(self.root): - name = relative_path[len(self.root) + 1:] - return name - - def _create_template(self, name): - path = os.path.join(self.root, name) - with open(path, "rb") as f: - template = Template(f.read(), name=name, loader=self) - return template - - -class DictLoader(BaseLoader): - """A template loader that loads from a dictionary.""" - def __init__(self, dict, **kwargs): - super(DictLoader, self).__init__(**kwargs) - self.dict = dict - - def resolve_path(self, name, parent_path=None): - if parent_path and not parent_path.startswith("<") and \ - not parent_path.startswith("/") and \ - not name.startswith("/"): - file_dir = posixpath.dirname(parent_path) - name = posixpath.normpath(posixpath.join(file_dir, name)) - return name - - def _create_template(self, name): - return Template(self.dict[name], name=name, loader=self) - - -class _Node(object): - def each_child(self): - return () - - def generate(self, writer): - raise NotImplementedError() - - def find_named_blocks(self, loader, named_blocks): - for child in self.each_child(): - child.find_named_blocks(loader, named_blocks) - - -class _File(_Node): - def __init__(self, template, body): - self.template = template - self.body = body - self.line = 0 - - def generate(self, writer): - writer.write_line("def _tt_execute():", self.line) - with writer.indent(): - writer.write_line("_tt_buffer = []", self.line) - writer.write_line("_tt_append = _tt_buffer.append", self.line) - self.body.generate(writer) - writer.write_line("return _tt_utf8('').join(_tt_buffer)", self.line) - - def each_child(self): - return (self.body,) - - -class _ChunkList(_Node): - def __init__(self, chunks): - self.chunks = chunks - - def generate(self, writer): - for chunk in self.chunks: - chunk.generate(writer) - - def each_child(self): - return self.chunks - - -class _NamedBlock(_Node): - def __init__(self, name, body, template, line): - self.name = name - self.body = body - self.template = template - self.line = line - - def each_child(self): - return (self.body,) - - def generate(self, writer): - block = writer.named_blocks[self.name] - with writer.include(block.template, self.line): - block.body.generate(writer) - - def find_named_blocks(self, loader, named_blocks): - named_blocks[self.name] = self - _Node.find_named_blocks(self, loader, named_blocks) - - -class _ExtendsBlock(_Node): - def __init__(self, name): - self.name = name - - -class _IncludeBlock(_Node): - def __init__(self, name, reader, line): - self.name = name - self.template_name = reader.name - self.line = line - - def find_named_blocks(self, loader, named_blocks): - included = loader.load(self.name, self.template_name) - included.file.find_named_blocks(loader, named_blocks) - - def generate(self, writer): - included = writer.loader.load(self.name, self.template_name) - with writer.include(included, self.line): - included.file.body.generate(writer) - - -class _ApplyBlock(_Node): - def __init__(self, method, line, body=None): - self.method = method - self.line = line - self.body = body - - def each_child(self): - return (self.body,) - - def generate(self, writer): - method_name = "_tt_apply%d" % writer.apply_counter - writer.apply_counter += 1 - writer.write_line("def %s():" % method_name, self.line) - with writer.indent(): - writer.write_line("_tt_buffer = []", self.line) - writer.write_line("_tt_append = _tt_buffer.append", self.line) - self.body.generate(writer) - writer.write_line("return _tt_utf8('').join(_tt_buffer)", self.line) - writer.write_line("_tt_append(_tt_utf8(%s(%s())))" % ( - self.method, method_name), self.line) - - -class _ControlBlock(_Node): - def __init__(self, statement, line, body=None): - self.statement = statement - self.line = line - self.body = body - - def each_child(self): - return (self.body,) - - def generate(self, writer): - writer.write_line("%s:" % self.statement, self.line) - with writer.indent(): - self.body.generate(writer) - # Just in case the body was empty - writer.write_line("pass", self.line) - - -class _IntermediateControlBlock(_Node): - def __init__(self, statement, line): - self.statement = statement - self.line = line - - def generate(self, writer): - # In case the previous block was empty - writer.write_line("pass", self.line) - writer.write_line("%s:" % self.statement, self.line, writer.indent_size() - 1) - - -class _Statement(_Node): - def __init__(self, statement, line): - self.statement = statement - self.line = line - - def generate(self, writer): - writer.write_line(self.statement, self.line) - - -class _Expression(_Node): - def __init__(self, expression, line, raw=False): - self.expression = expression - self.line = line - self.raw = raw - - def generate(self, writer): - writer.write_line("_tt_tmp = %s" % self.expression, self.line) - writer.write_line("if isinstance(_tt_tmp, _tt_string_types):" - " _tt_tmp = _tt_utf8(_tt_tmp)", self.line) - writer.write_line("else: _tt_tmp = _tt_utf8(str(_tt_tmp))", self.line) - if not self.raw and writer.current_template.autoescape is not None: - # In python3 functions like xhtml_escape return unicode, - # so we have to convert to utf8 again. - writer.write_line("_tt_tmp = _tt_utf8(%s(_tt_tmp))" % - writer.current_template.autoescape, self.line) - writer.write_line("_tt_append(_tt_tmp)", self.line) - - -class _Module(_Expression): - def __init__(self, expression, line): - super(_Module, self).__init__("_tt_modules." + expression, line, - raw=True) - - -class _Text(_Node): - def __init__(self, value, line, whitespace): - self.value = value - self.line = line - self.whitespace = whitespace - - def generate(self, writer): - value = self.value - - # Compress whitespace if requested, with a crude heuristic to avoid - # altering preformatted whitespace. - if "
    " not in value:
-            value = filter_whitespace(self.whitespace, value)
-
-        if value:
-            writer.write_line('_tt_append(%r)' % escape.utf8(value), self.line)
-
-
-class ParseError(Exception):
-    """Raised for template syntax errors.
-
-    ``ParseError`` instances have ``filename`` and ``lineno`` attributes
-    indicating the position of the error.
-
-    .. versionchanged:: 4.3
-       Added ``filename`` and ``lineno`` attributes.
-    """
-    def __init__(self, message, filename=None, lineno=0):
-        self.message = message
-        # The names "filename" and "lineno" are chosen for consistency
-        # with python SyntaxError.
-        self.filename = filename
-        self.lineno = lineno
-
-    def __str__(self):
-        return '%s at %s:%d' % (self.message, self.filename, self.lineno)
-
-
-class _CodeWriter(object):
-    def __init__(self, file, named_blocks, loader, current_template):
-        self.file = file
-        self.named_blocks = named_blocks
-        self.loader = loader
-        self.current_template = current_template
-        self.apply_counter = 0
-        self.include_stack = []
-        self._indent = 0
-
-    def indent_size(self):
-        return self._indent
-
-    def indent(self):
-        class Indenter(object):
-            def __enter__(_):
-                self._indent += 1
-                return self
-
-            def __exit__(_, *args):
-                assert self._indent > 0
-                self._indent -= 1
-
-        return Indenter()
-
-    def include(self, template, line):
-        self.include_stack.append((self.current_template, line))
-        self.current_template = template
-
-        class IncludeTemplate(object):
-            def __enter__(_):
-                return self
-
-            def __exit__(_, *args):
-                self.current_template = self.include_stack.pop()[0]
-
-        return IncludeTemplate()
-
-    def write_line(self, line, line_number, indent=None):
-        if indent is None:
-            indent = self._indent
-        line_comment = '  # %s:%d' % (self.current_template.name, line_number)
-        if self.include_stack:
-            ancestors = ["%s:%d" % (tmpl.name, lineno)
-                         for (tmpl, lineno) in self.include_stack]
-            line_comment += ' (via %s)' % ', '.join(reversed(ancestors))
-        print("    " * indent + line + line_comment, file=self.file)
-
-
-class _TemplateReader(object):
-    def __init__(self, name, text, whitespace):
-        self.name = name
-        self.text = text
-        self.whitespace = whitespace
-        self.line = 1
-        self.pos = 0
-
-    def find(self, needle, start=0, end=None):
-        assert start >= 0, start
-        pos = self.pos
-        start += pos
-        if end is None:
-            index = self.text.find(needle, start)
-        else:
-            end += pos
-            assert end >= start
-            index = self.text.find(needle, start, end)
-        if index != -1:
-            index -= pos
-        return index
-
-    def consume(self, count=None):
-        if count is None:
-            count = len(self.text) - self.pos
-        newpos = self.pos + count
-        self.line += self.text.count("\n", self.pos, newpos)
-        s = self.text[self.pos:newpos]
-        self.pos = newpos
-        return s
-
-    def remaining(self):
-        return len(self.text) - self.pos
-
-    def __len__(self):
-        return self.remaining()
-
-    def __getitem__(self, key):
-        if type(key) is slice:
-            size = len(self)
-            start, stop, step = key.indices(size)
-            if start is None:
-                start = self.pos
-            else:
-                start += self.pos
-            if stop is not None:
-                stop += self.pos
-            return self.text[slice(start, stop, step)]
-        elif key < 0:
-            return self.text[key]
-        else:
-            return self.text[self.pos + key]
-
-    def __str__(self):
-        return self.text[self.pos:]
-
-    def raise_parse_error(self, msg):
-        raise ParseError(msg, self.name, self.line)
-
-
-def _format_code(code):
-    lines = code.splitlines()
-    format = "%%%dd  %%s\n" % len(repr(len(lines) + 1))
-    return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)])
-
-
-def _parse(reader, template, in_block=None, in_loop=None):
-    body = _ChunkList([])
-    while True:
-        # Find next template directive
-        curly = 0
-        while True:
-            curly = reader.find("{", curly)
-            if curly == -1 or curly + 1 == reader.remaining():
-                # EOF
-                if in_block:
-                    reader.raise_parse_error(
-                        "Missing {%% end %%} block for %s" % in_block)
-                body.chunks.append(_Text(reader.consume(), reader.line,
-                                         reader.whitespace))
-                return body
-            # If the first curly brace is not the start of a special token,
-            # start searching from the character after it
-            if reader[curly + 1] not in ("{", "%", "#"):
-                curly += 1
-                continue
-            # When there are more than 2 curlies in a row, use the
-            # innermost ones.  This is useful when generating languages
-            # like latex where curlies are also meaningful
-            if (curly + 2 < reader.remaining() and
-                    reader[curly + 1] == '{' and reader[curly + 2] == '{'):
-                curly += 1
-                continue
-            break
-
-        # Append any text before the special token
-        if curly > 0:
-            cons = reader.consume(curly)
-            body.chunks.append(_Text(cons, reader.line,
-                                     reader.whitespace))
-
-        start_brace = reader.consume(2)
-        line = reader.line
-
-        # Template directives may be escaped as "{{!" or "{%!".
-        # In this case output the braces and consume the "!".
-        # This is especially useful in conjunction with jquery templates,
-        # which also use double braces.
-        if reader.remaining() and reader[0] == "!":
-            reader.consume(1)
-            body.chunks.append(_Text(start_brace, line,
-                                     reader.whitespace))
-            continue
-
-        # Comment
-        if start_brace == "{#":
-            end = reader.find("#}")
-            if end == -1:
-                reader.raise_parse_error("Missing end comment #}")
-            contents = reader.consume(end).strip()
-            reader.consume(2)
-            continue
-
-        # Expression
-        if start_brace == "{{":
-            end = reader.find("}}")
-            if end == -1:
-                reader.raise_parse_error("Missing end expression }}")
-            contents = reader.consume(end).strip()
-            reader.consume(2)
-            if not contents:
-                reader.raise_parse_error("Empty expression")
-            body.chunks.append(_Expression(contents, line))
-            continue
-
-        # Block
-        assert start_brace == "{%", start_brace
-        end = reader.find("%}")
-        if end == -1:
-            reader.raise_parse_error("Missing end block %}")
-        contents = reader.consume(end).strip()
-        reader.consume(2)
-        if not contents:
-            reader.raise_parse_error("Empty block tag ({% %})")
-
-        operator, space, suffix = contents.partition(" ")
-        suffix = suffix.strip()
-
-        # Intermediate ("else", "elif", etc) blocks
-        intermediate_blocks = {
-            "else": set(["if", "for", "while", "try"]),
-            "elif": set(["if"]),
-            "except": set(["try"]),
-            "finally": set(["try"]),
-        }
-        allowed_parents = intermediate_blocks.get(operator)
-        if allowed_parents is not None:
-            if not in_block:
-                reader.raise_parse_error("%s outside %s block" %
-                                         (operator, allowed_parents))
-            if in_block not in allowed_parents:
-                reader.raise_parse_error(
-                    "%s block cannot be attached to %s block" %
-                    (operator, in_block))
-            body.chunks.append(_IntermediateControlBlock(contents, line))
-            continue
-
-        # End tag
-        elif operator == "end":
-            if not in_block:
-                reader.raise_parse_error("Extra {% end %} block")
-            return body
-
-        elif operator in ("extends", "include", "set", "import", "from",
-                          "comment", "autoescape", "whitespace", "raw",
-                          "module"):
-            if operator == "comment":
-                continue
-            if operator == "extends":
-                suffix = suffix.strip('"').strip("'")
-                if not suffix:
-                    reader.raise_parse_error("extends missing file path")
-                block = _ExtendsBlock(suffix)
-            elif operator in ("import", "from"):
-                if not suffix:
-                    reader.raise_parse_error("import missing statement")
-                block = _Statement(contents, line)
-            elif operator == "include":
-                suffix = suffix.strip('"').strip("'")
-                if not suffix:
-                    reader.raise_parse_error("include missing file path")
-                block = _IncludeBlock(suffix, reader, line)
-            elif operator == "set":
-                if not suffix:
-                    reader.raise_parse_error("set missing statement")
-                block = _Statement(suffix, line)
-            elif operator == "autoescape":
-                fn = suffix.strip()
-                if fn == "None":
-                    fn = None
-                template.autoescape = fn
-                continue
-            elif operator == "whitespace":
-                mode = suffix.strip()
-                # Validate the selected mode
-                filter_whitespace(mode, '')
-                reader.whitespace = mode
-                continue
-            elif operator == "raw":
-                block = _Expression(suffix, line, raw=True)
-            elif operator == "module":
-                block = _Module(suffix, line)
-            body.chunks.append(block)
-            continue
-
-        elif operator in ("apply", "block", "try", "if", "for", "while"):
-            # parse inner body recursively
-            if operator in ("for", "while"):
-                block_body = _parse(reader, template, operator, operator)
-            elif operator == "apply":
-                # apply creates a nested function so syntactically it's not
-                # in the loop.
-                block_body = _parse(reader, template, operator, None)
-            else:
-                block_body = _parse(reader, template, operator, in_loop)
-
-            if operator == "apply":
-                if not suffix:
-                    reader.raise_parse_error("apply missing method name")
-                block = _ApplyBlock(suffix, line, block_body)
-            elif operator == "block":
-                if not suffix:
-                    reader.raise_parse_error("block missing name")
-                block = _NamedBlock(suffix, block_body, template, line)
-            else:
-                block = _ControlBlock(contents, line, block_body)
-            body.chunks.append(block)
-            continue
-
-        elif operator in ("break", "continue"):
-            if not in_loop:
-                reader.raise_parse_error("%s outside %s block" %
-                                         (operator, set(["for", "while"])))
-            body.chunks.append(_Statement(contents, line))
-            continue
-
-        else:
-            reader.raise_parse_error("unknown operator: %r" % operator)
diff --git a/lib/tornado/test/__main__.py b/lib/tornado/test/__main__.py
deleted file mode 100755
index c78478cb..00000000
--- a/lib/tornado/test/__main__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-"""Shim to allow python -m tornado.test.
-
-This only works in python 2.7+.
-"""
-from __future__ import absolute_import, division, print_function
-
-from tornado.test.runtests import all, main
-
-# tornado.testing.main autodiscovery relies on 'all' being present in
-# the main module, so import it here even though it is not used directly.
-# The following line prevents a pyflakes warning.
-all = all
-
-main()
diff --git a/lib/tornado/test/asyncio_test.py b/lib/tornado/test/asyncio_test.py
deleted file mode 100755
index a7c75649..00000000
--- a/lib/tornado/test/asyncio_test.py
+++ /dev/null
@@ -1,206 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from __future__ import absolute_import, division, print_function
-
-from concurrent.futures import ThreadPoolExecutor
-from tornado import gen
-from tornado.ioloop import IOLoop
-from tornado.testing import AsyncTestCase, gen_test
-from tornado.test.util import unittest, skipBefore33, skipBefore35, exec_test
-
-try:
-    from tornado.platform.asyncio import asyncio
-except ImportError:
-    asyncio = None
-else:
-    from tornado.platform.asyncio import AsyncIOLoop, to_asyncio_future, AnyThreadEventLoopPolicy
-    # This is used in dynamically-evaluated code, so silence pyflakes.
-    to_asyncio_future
-
-
-@unittest.skipIf(asyncio is None, "asyncio module not present")
-class AsyncIOLoopTest(AsyncTestCase):
-    def get_new_ioloop(self):
-        io_loop = AsyncIOLoop()
-        return io_loop
-
-    def test_asyncio_callback(self):
-        # Basic test that the asyncio loop is set up correctly.
-        asyncio.get_event_loop().call_soon(self.stop)
-        self.wait()
-
-    @gen_test
-    def test_asyncio_future(self):
-        # Test that we can yield an asyncio future from a tornado coroutine.
-        # Without 'yield from', we must wrap coroutines in ensure_future,
-        # which was introduced during Python 3.4, deprecating the prior "async".
-        if hasattr(asyncio, 'ensure_future'):
-            ensure_future = asyncio.ensure_future
-        else:
-            # async is a reserved word in Python 3.7
-            ensure_future = getattr(asyncio, 'async')
-
-        x = yield ensure_future(
-            asyncio.get_event_loop().run_in_executor(None, lambda: 42))
-        self.assertEqual(x, 42)
-
-    @skipBefore33
-    @gen_test
-    def test_asyncio_yield_from(self):
-        # Test that we can use asyncio coroutines with 'yield from'
-        # instead of asyncio.async(). This requires python 3.3 syntax.
-        namespace = exec_test(globals(), locals(), """
-        @gen.coroutine
-        def f():
-            event_loop = asyncio.get_event_loop()
-            x = yield from event_loop.run_in_executor(None, lambda: 42)
-            return x
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-
-    @skipBefore35
-    def test_asyncio_adapter(self):
-        # This test demonstrates that when using the asyncio coroutine
-        # runner (i.e. run_until_complete), the to_asyncio_future
-        # adapter is needed. No adapter is needed in the other direction,
-        # as demonstrated by other tests in the package.
-        @gen.coroutine
-        def tornado_coroutine():
-            yield gen.moment
-            raise gen.Return(42)
-        native_coroutine_without_adapter = exec_test(globals(), locals(), """
-        async def native_coroutine_without_adapter():
-            return await tornado_coroutine()
-        """)["native_coroutine_without_adapter"]
-
-        native_coroutine_with_adapter = exec_test(globals(), locals(), """
-        async def native_coroutine_with_adapter():
-            return await to_asyncio_future(tornado_coroutine())
-        """)["native_coroutine_with_adapter"]
-
-        # Use the adapter, but two degrees from the tornado coroutine.
-        native_coroutine_with_adapter2 = exec_test(globals(), locals(), """
-        async def native_coroutine_with_adapter2():
-            return await to_asyncio_future(native_coroutine_without_adapter())
-        """)["native_coroutine_with_adapter2"]
-
-        # Tornado supports native coroutines both with and without adapters
-        self.assertEqual(
-            self.io_loop.run_sync(native_coroutine_without_adapter),
-            42)
-        self.assertEqual(
-            self.io_loop.run_sync(native_coroutine_with_adapter),
-            42)
-        self.assertEqual(
-            self.io_loop.run_sync(native_coroutine_with_adapter2),
-            42)
-
-        # Asyncio only supports coroutines that yield asyncio-compatible
-        # Futures (which our Future is since 5.0).
-        self.assertEqual(
-            asyncio.get_event_loop().run_until_complete(
-                native_coroutine_without_adapter()),
-            42)
-        self.assertEqual(
-            asyncio.get_event_loop().run_until_complete(
-                native_coroutine_with_adapter()),
-            42)
-        self.assertEqual(
-            asyncio.get_event_loop().run_until_complete(
-                native_coroutine_with_adapter2()),
-            42)
-
-
-@unittest.skipIf(asyncio is None, "asyncio module not present")
-class LeakTest(unittest.TestCase):
-    def setUp(self):
-        # Trigger a cleanup of the mapping so we start with a clean slate.
-        AsyncIOLoop().close()
-        # If we don't clean up after ourselves other tests may fail on
-        # py34.
-        self.orig_policy = asyncio.get_event_loop_policy()
-        asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
-
-    def tearDown(self):
-        asyncio.get_event_loop().close()
-        asyncio.set_event_loop_policy(self.orig_policy)
-
-    def test_ioloop_close_leak(self):
-        orig_count = len(IOLoop._ioloop_for_asyncio)
-        for i in range(10):
-            # Create and close an AsyncIOLoop using Tornado interfaces.
-            loop = AsyncIOLoop()
-            loop.close()
-        new_count = len(IOLoop._ioloop_for_asyncio) - orig_count
-        self.assertEqual(new_count, 0)
-
-    def test_asyncio_close_leak(self):
-        orig_count = len(IOLoop._ioloop_for_asyncio)
-        for i in range(10):
-            # Create and close an AsyncIOMainLoop using asyncio interfaces.
-            loop = asyncio.new_event_loop()
-            loop.call_soon(IOLoop.current)
-            loop.call_soon(loop.stop)
-            loop.run_forever()
-            loop.close()
-        new_count = len(IOLoop._ioloop_for_asyncio) - orig_count
-        # Because the cleanup is run on new loop creation, we have one
-        # dangling entry in the map (but only one).
-        self.assertEqual(new_count, 1)
-
-
-@unittest.skipIf(asyncio is None, "asyncio module not present")
-class AnyThreadEventLoopPolicyTest(unittest.TestCase):
-    def setUp(self):
-        self.orig_policy = asyncio.get_event_loop_policy()
-        self.executor = ThreadPoolExecutor(1)
-
-    def tearDown(self):
-        asyncio.set_event_loop_policy(self.orig_policy)
-        self.executor.shutdown()
-
-    def get_event_loop_on_thread(self):
-        def get_and_close_event_loop():
-            """Get the event loop. Close it if one is returned.
-
-            Returns the (closed) event loop. This is a silly thing
-            to do and leaves the thread in a broken state, but it's
-            enough for this test. Closing the loop avoids resource
-            leak warnings.
-            """
-            loop = asyncio.get_event_loop()
-            loop.close()
-            return loop
-        future = self.executor.submit(get_and_close_event_loop)
-        return future.result()
-
-    def run_policy_test(self, accessor, expected_type):
-        # With the default policy, non-main threads don't get an event
-        # loop.
-        self.assertRaises((RuntimeError, AssertionError),
-                          self.executor.submit(accessor).result)
-        # Set the policy and we can get a loop.
-        asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
-        self.assertIsInstance(
-            self.executor.submit(accessor).result(),
-            expected_type)
-        # Clean up to silence leak warnings. Always use asyncio since
-        # IOLoop doesn't (currently) close the underlying loop.
-        self.executor.submit(lambda: asyncio.get_event_loop().close()).result()
-
-    def test_asyncio_accessor(self):
-        self.run_policy_test(asyncio.get_event_loop, asyncio.AbstractEventLoop)
-
-    def test_tornado_accessor(self):
-        self.run_policy_test(IOLoop.current, IOLoop)
diff --git a/lib/tornado/test/auth_test.py b/lib/tornado/test/auth_test.py
deleted file mode 100755
index 14bc3353..00000000
--- a/lib/tornado/test/auth_test.py
+++ /dev/null
@@ -1,735 +0,0 @@
-# These tests do not currently do much to verify the correct implementation
-# of the openid/oauth protocols, they just exercise the major code paths
-# and ensure that it doesn't blow up (e.g. with unicode/bytes issues in
-# python 3)
-
-
-from __future__ import absolute_import, division, print_function
-
-import unittest
-import warnings
-
-from tornado.auth import (
-    AuthError, OpenIdMixin, OAuthMixin, OAuth2Mixin,
-    GoogleOAuth2Mixin, FacebookGraphMixin, TwitterMixin,
-)
-from tornado.concurrent import Future
-from tornado.escape import json_decode
-from tornado import gen
-from tornado.httputil import url_concat
-from tornado.log import gen_log, app_log
-from tornado.testing import AsyncHTTPTestCase, ExpectLog
-from tornado.test.util import ignore_deprecation
-from tornado.web import RequestHandler, Application, asynchronous, HTTPError
-
-try:
-    from unittest import mock
-except ImportError:
-    mock = None
-
-
-class OpenIdClientLoginHandlerLegacy(RequestHandler, OpenIdMixin):
-    def initialize(self, test):
-        self._OPENID_ENDPOINT = test.get_url('/openid/server/authenticate')
-
-    with ignore_deprecation():
-        @asynchronous
-        def get(self):
-            if self.get_argument('openid.mode', None):
-                with warnings.catch_warnings():
-                    warnings.simplefilter('ignore', DeprecationWarning)
-                    self.get_authenticated_user(
-                        self.on_user, http_client=self.settings['http_client'])
-                    return
-            res = self.authenticate_redirect()
-            assert isinstance(res, Future)
-            assert res.done()
-
-    def on_user(self, user):
-        if user is None:
-            raise Exception("user is None")
-        self.finish(user)
-
-
-class OpenIdClientLoginHandler(RequestHandler, OpenIdMixin):
-    def initialize(self, test):
-        self._OPENID_ENDPOINT = test.get_url('/openid/server/authenticate')
-
-    @gen.coroutine
-    def get(self):
-        if self.get_argument('openid.mode', None):
-            user = yield self.get_authenticated_user(http_client=self.settings['http_client'])
-            if user is None:
-                raise Exception("user is None")
-            self.finish(user)
-            return
-        res = self.authenticate_redirect()
-        assert isinstance(res, Future)
-        assert res.done()
-
-
-class OpenIdServerAuthenticateHandler(RequestHandler):
-    def post(self):
-        if self.get_argument('openid.mode') != 'check_authentication':
-            raise Exception("incorrect openid.mode %r")
-        self.write('is_valid:true')
-
-
-class OAuth1ClientLoginHandlerLegacy(RequestHandler, OAuthMixin):
-    def initialize(self, test, version):
-        self._OAUTH_VERSION = version
-        self._OAUTH_REQUEST_TOKEN_URL = test.get_url('/oauth1/server/request_token')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth1/server/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/oauth1/server/access_token')
-
-    def _oauth_consumer_token(self):
-        return dict(key='asdf', secret='qwer')
-
-    with ignore_deprecation():
-        @asynchronous
-        def get(self):
-            if self.get_argument('oauth_token', None):
-                with warnings.catch_warnings():
-                    warnings.simplefilter('ignore', DeprecationWarning)
-                    self.get_authenticated_user(
-                        self.on_user, http_client=self.settings['http_client'])
-                return
-            res = self.authorize_redirect(http_client=self.settings['http_client'])
-            assert isinstance(res, Future)
-
-    def on_user(self, user):
-        if user is None:
-            raise Exception("user is None")
-        self.finish(user)
-
-    def _oauth_get_user(self, access_token, callback):
-        if self.get_argument('fail_in_get_user', None):
-            raise Exception("failing in get_user")
-        if access_token != dict(key='uiop', secret='5678'):
-            raise Exception("incorrect access token %r" % access_token)
-        callback(dict(email='foo@example.com'))
-
-
-class OAuth1ClientLoginHandler(RequestHandler, OAuthMixin):
-    def initialize(self, test, version):
-        self._OAUTH_VERSION = version
-        self._OAUTH_REQUEST_TOKEN_URL = test.get_url('/oauth1/server/request_token')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth1/server/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/oauth1/server/access_token')
-
-    def _oauth_consumer_token(self):
-        return dict(key='asdf', secret='qwer')
-
-    @gen.coroutine
-    def get(self):
-        if self.get_argument('oauth_token', None):
-            user = yield self.get_authenticated_user(http_client=self.settings['http_client'])
-            if user is None:
-                raise Exception("user is None")
-            self.finish(user)
-            return
-        yield self.authorize_redirect(http_client=self.settings['http_client'])
-
-    @gen.coroutine
-    def _oauth_get_user_future(self, access_token):
-        if self.get_argument('fail_in_get_user', None):
-            raise Exception("failing in get_user")
-        if access_token != dict(key='uiop', secret='5678'):
-            raise Exception("incorrect access token %r" % access_token)
-        return dict(email='foo@example.com')
-
-
-class OAuth1ClientLoginCoroutineHandler(OAuth1ClientLoginHandler):
-    """Replaces OAuth1ClientLoginCoroutineHandler's get() with a coroutine."""
-    @gen.coroutine
-    def get(self):
-        if self.get_argument('oauth_token', None):
-            # Ensure that any exceptions are set on the returned Future,
-            # not simply thrown into the surrounding StackContext.
-            try:
-                yield self.get_authenticated_user()
-            except Exception as e:
-                self.set_status(503)
-                self.write("got exception: %s" % e)
-        else:
-            yield self.authorize_redirect()
-
-
-class OAuth1ClientRequestParametersHandler(RequestHandler, OAuthMixin):
-    def initialize(self, version):
-        self._OAUTH_VERSION = version
-
-    def _oauth_consumer_token(self):
-        return dict(key='asdf', secret='qwer')
-
-    def get(self):
-        params = self._oauth_request_parameters(
-            'http://www.example.com/api/asdf',
-            dict(key='uiop', secret='5678'),
-            parameters=dict(foo='bar'))
-        self.write(params)
-
-
-class OAuth1ServerRequestTokenHandler(RequestHandler):
-    def get(self):
-        self.write('oauth_token=zxcv&oauth_token_secret=1234')
-
-
-class OAuth1ServerAccessTokenHandler(RequestHandler):
-    def get(self):
-        self.write('oauth_token=uiop&oauth_token_secret=5678')
-
-
-class OAuth2ClientLoginHandler(RequestHandler, OAuth2Mixin):
-    def initialize(self, test):
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth2/server/authorize')
-
-    def get(self):
-        res = self.authorize_redirect()
-        assert isinstance(res, Future)
-        assert res.done()
-
-
-class FacebookClientLoginHandler(RequestHandler, FacebookGraphMixin):
-    def initialize(self, test):
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/facebook/server/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/facebook/server/access_token')
-        self._FACEBOOK_BASE_URL = test.get_url('/facebook/server')
-
-    @gen.coroutine
-    def get(self):
-        if self.get_argument("code", None):
-            user = yield self.get_authenticated_user(
-                redirect_uri=self.request.full_url(),
-                client_id=self.settings["facebook_api_key"],
-                client_secret=self.settings["facebook_secret"],
-                code=self.get_argument("code"))
-            self.write(user)
-        else:
-            yield self.authorize_redirect(
-                redirect_uri=self.request.full_url(),
-                client_id=self.settings["facebook_api_key"],
-                extra_params={"scope": "read_stream,offline_access"})
-
-
-class FacebookServerAccessTokenHandler(RequestHandler):
-    def get(self):
-        self.write(dict(access_token="asdf", expires_in=3600))
-
-
-class FacebookServerMeHandler(RequestHandler):
-    def get(self):
-        self.write('{}')
-
-
-class TwitterClientHandler(RequestHandler, TwitterMixin):
-    def initialize(self, test):
-        self._OAUTH_REQUEST_TOKEN_URL = test.get_url('/oauth1/server/request_token')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/twitter/server/access_token')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth1/server/authorize')
-        self._OAUTH_AUTHENTICATE_URL = test.get_url('/twitter/server/authenticate')
-        self._TWITTER_BASE_URL = test.get_url('/twitter/api')
-
-    def get_auth_http_client(self):
-        return self.settings['http_client']
-
-
-class TwitterClientLoginHandlerLegacy(TwitterClientHandler):
-    with ignore_deprecation():
-        @asynchronous
-        def get(self):
-            if self.get_argument("oauth_token", None):
-                self.get_authenticated_user(self.on_user)
-                return
-            self.authorize_redirect()
-
-    def on_user(self, user):
-        if user is None:
-            raise Exception("user is None")
-        self.finish(user)
-
-
-class TwitterClientLoginHandler(TwitterClientHandler):
-    @gen.coroutine
-    def get(self):
-        if self.get_argument("oauth_token", None):
-            user = yield self.get_authenticated_user()
-            if user is None:
-                raise Exception("user is None")
-            self.finish(user)
-            return
-        yield self.authorize_redirect()
-
-
-class TwitterClientAuthenticateHandler(TwitterClientHandler):
-    # Like TwitterClientLoginHandler, but uses authenticate_redirect
-    # instead of authorize_redirect.
-    @gen.coroutine
-    def get(self):
-        if self.get_argument("oauth_token", None):
-            user = yield self.get_authenticated_user()
-            if user is None:
-                raise Exception("user is None")
-            self.finish(user)
-            return
-        yield self.authenticate_redirect()
-
-
-class TwitterClientLoginGenEngineHandler(TwitterClientHandler):
-    with ignore_deprecation():
-        @asynchronous
-        @gen.engine
-        def get(self):
-            if self.get_argument("oauth_token", None):
-                user = yield self.get_authenticated_user()
-                self.finish(user)
-            else:
-                # Old style: with @gen.engine we can ignore the Future from
-                # authorize_redirect.
-                self.authorize_redirect()
-
-
-class TwitterClientLoginGenCoroutineHandler(TwitterClientHandler):
-    @gen.coroutine
-    def get(self):
-        if self.get_argument("oauth_token", None):
-            user = yield self.get_authenticated_user()
-            self.finish(user)
-        else:
-            # New style: with @gen.coroutine the result must be yielded
-            # or else the request will be auto-finished too soon.
-            yield self.authorize_redirect()
-
-
-class TwitterClientShowUserHandlerLegacy(TwitterClientHandler):
-    with ignore_deprecation():
-        @asynchronous
-        @gen.engine
-        def get(self):
-            # TODO: would be nice to go through the login flow instead of
-            # cheating with a hard-coded access token.
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                response = yield gen.Task(self.twitter_request,
-                                          '/users/show/%s' % self.get_argument('name'),
-                                          access_token=dict(key='hjkl', secret='vbnm'))
-            if response is None:
-                self.set_status(500)
-                self.finish('error from twitter request')
-            else:
-                self.finish(response)
-
-
-class TwitterClientShowUserHandler(TwitterClientHandler):
-    @gen.coroutine
-    def get(self):
-        # TODO: would be nice to go through the login flow instead of
-        # cheating with a hard-coded access token.
-        try:
-            response = yield self.twitter_request(
-                '/users/show/%s' % self.get_argument('name'),
-                access_token=dict(key='hjkl', secret='vbnm'))
-        except AuthError:
-            self.set_status(500)
-            self.finish('error from twitter request')
-        else:
-            self.finish(response)
-
-
-class TwitterServerAccessTokenHandler(RequestHandler):
-    def get(self):
-        self.write('oauth_token=hjkl&oauth_token_secret=vbnm&screen_name=foo')
-
-
-class TwitterServerShowUserHandler(RequestHandler):
-    def get(self, screen_name):
-        if screen_name == 'error':
-            raise HTTPError(500)
-        assert 'oauth_nonce' in self.request.arguments
-        assert 'oauth_timestamp' in self.request.arguments
-        assert 'oauth_signature' in self.request.arguments
-        assert self.get_argument('oauth_consumer_key') == 'test_twitter_consumer_key'
-        assert self.get_argument('oauth_signature_method') == 'HMAC-SHA1'
-        assert self.get_argument('oauth_version') == '1.0'
-        assert self.get_argument('oauth_token') == 'hjkl'
-        self.write(dict(screen_name=screen_name, name=screen_name.capitalize()))
-
-
-class TwitterServerVerifyCredentialsHandler(RequestHandler):
-    def get(self):
-        assert 'oauth_nonce' in self.request.arguments
-        assert 'oauth_timestamp' in self.request.arguments
-        assert 'oauth_signature' in self.request.arguments
-        assert self.get_argument('oauth_consumer_key') == 'test_twitter_consumer_key'
-        assert self.get_argument('oauth_signature_method') == 'HMAC-SHA1'
-        assert self.get_argument('oauth_version') == '1.0'
-        assert self.get_argument('oauth_token') == 'hjkl'
-        self.write(dict(screen_name='foo', name='Foo'))
-
-
-class AuthTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application(
-            [
-                # test endpoints
-                ('/legacy/openid/client/login', OpenIdClientLoginHandlerLegacy, dict(test=self)),
-                ('/openid/client/login', OpenIdClientLoginHandler, dict(test=self)),
-                ('/legacy/oauth10/client/login', OAuth1ClientLoginHandlerLegacy,
-                 dict(test=self, version='1.0')),
-                ('/oauth10/client/login', OAuth1ClientLoginHandler,
-                 dict(test=self, version='1.0')),
-                ('/oauth10/client/request_params',
-                 OAuth1ClientRequestParametersHandler,
-                 dict(version='1.0')),
-                ('/legacy/oauth10a/client/login', OAuth1ClientLoginHandlerLegacy,
-                 dict(test=self, version='1.0a')),
-                ('/oauth10a/client/login', OAuth1ClientLoginHandler,
-                 dict(test=self, version='1.0a')),
-                ('/oauth10a/client/login_coroutine',
-                 OAuth1ClientLoginCoroutineHandler,
-                 dict(test=self, version='1.0a')),
-                ('/oauth10a/client/request_params',
-                 OAuth1ClientRequestParametersHandler,
-                 dict(version='1.0a')),
-                ('/oauth2/client/login', OAuth2ClientLoginHandler, dict(test=self)),
-
-                ('/facebook/client/login', FacebookClientLoginHandler, dict(test=self)),
-
-                ('/legacy/twitter/client/login', TwitterClientLoginHandlerLegacy, dict(test=self)),
-                ('/twitter/client/login', TwitterClientLoginHandler, dict(test=self)),
-                ('/twitter/client/authenticate', TwitterClientAuthenticateHandler, dict(test=self)),
-                ('/twitter/client/login_gen_engine',
-                 TwitterClientLoginGenEngineHandler, dict(test=self)),
-                ('/twitter/client/login_gen_coroutine',
-                 TwitterClientLoginGenCoroutineHandler, dict(test=self)),
-                ('/legacy/twitter/client/show_user',
-                 TwitterClientShowUserHandlerLegacy, dict(test=self)),
-                ('/twitter/client/show_user',
-                 TwitterClientShowUserHandler, dict(test=self)),
-
-                # simulated servers
-                ('/openid/server/authenticate', OpenIdServerAuthenticateHandler),
-                ('/oauth1/server/request_token', OAuth1ServerRequestTokenHandler),
-                ('/oauth1/server/access_token', OAuth1ServerAccessTokenHandler),
-
-                ('/facebook/server/access_token', FacebookServerAccessTokenHandler),
-                ('/facebook/server/me', FacebookServerMeHandler),
-                ('/twitter/server/access_token', TwitterServerAccessTokenHandler),
-                (r'/twitter/api/users/show/(.*)\.json', TwitterServerShowUserHandler),
-                (r'/twitter/api/account/verify_credentials\.json',
-                 TwitterServerVerifyCredentialsHandler),
-            ],
-            http_client=self.http_client,
-            twitter_consumer_key='test_twitter_consumer_key',
-            twitter_consumer_secret='test_twitter_consumer_secret',
-            facebook_api_key='test_facebook_api_key',
-            facebook_secret='test_facebook_secret')
-
-    def test_openid_redirect_legacy(self):
-        response = self.fetch('/legacy/openid/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(
-            '/openid/server/authenticate?' in response.headers['Location'])
-
-    def test_openid_get_user_legacy(self):
-        response = self.fetch('/legacy/openid/client/login?openid.mode=blah'
-                              '&openid.ns.ax=http://openid.net/srv/ax/1.0'
-                              '&openid.ax.type.email=http://axschema.org/contact/email'
-                              '&openid.ax.value.email=foo@example.com')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed["email"], "foo@example.com")
-
-    def test_openid_redirect(self):
-        response = self.fetch('/openid/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(
-            '/openid/server/authenticate?' in response.headers['Location'])
-
-    def test_openid_get_user(self):
-        response = self.fetch('/openid/client/login?openid.mode=blah'
-                              '&openid.ns.ax=http://openid.net/srv/ax/1.0'
-                              '&openid.ax.type.email=http://axschema.org/contact/email'
-                              '&openid.ax.value.email=foo@example.com')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed["email"], "foo@example.com")
-
-    def test_oauth10_redirect_legacy(self):
-        response = self.fetch('/legacy/oauth10/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_oauth10_redirect(self):
-        response = self.fetch('/oauth10/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_oauth10_get_user_legacy(self):
-        with ignore_deprecation():
-            response = self.fetch(
-                '/legacy/oauth10/client/login?oauth_token=zxcv',
-                headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['email'], 'foo@example.com')
-        self.assertEqual(parsed['access_token'], dict(key='uiop', secret='5678'))
-
-    def test_oauth10_get_user(self):
-        response = self.fetch(
-            '/oauth10/client/login?oauth_token=zxcv',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['email'], 'foo@example.com')
-        self.assertEqual(parsed['access_token'], dict(key='uiop', secret='5678'))
-
-    def test_oauth10_request_parameters(self):
-        response = self.fetch('/oauth10/client/request_params')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['oauth_consumer_key'], 'asdf')
-        self.assertEqual(parsed['oauth_token'], 'uiop')
-        self.assertTrue('oauth_nonce' in parsed)
-        self.assertTrue('oauth_signature' in parsed)
-
-    def test_oauth10a_redirect_legacy(self):
-        response = self.fetch('/legacy/oauth10a/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_oauth10a_get_user_legacy(self):
-        with ignore_deprecation():
-            response = self.fetch(
-                '/legacy/oauth10a/client/login?oauth_token=zxcv',
-                headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['email'], 'foo@example.com')
-        self.assertEqual(parsed['access_token'], dict(key='uiop', secret='5678'))
-
-    def test_oauth10a_redirect(self):
-        response = self.fetch('/oauth10a/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    @unittest.skipIf(mock is None, 'mock package not present')
-    def test_oauth10a_redirect_error(self):
-        with mock.patch.object(OAuth1ServerRequestTokenHandler, 'get') as get:
-            get.side_effect = Exception("boom")
-            with ExpectLog(app_log, "Uncaught exception"):
-                response = self.fetch('/oauth10a/client/login', follow_redirects=False)
-            self.assertEqual(response.code, 500)
-
-    def test_oauth10a_get_user(self):
-        response = self.fetch(
-            '/oauth10a/client/login?oauth_token=zxcv',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['email'], 'foo@example.com')
-        self.assertEqual(parsed['access_token'], dict(key='uiop', secret='5678'))
-
-    def test_oauth10a_request_parameters(self):
-        response = self.fetch('/oauth10a/client/request_params')
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed['oauth_consumer_key'], 'asdf')
-        self.assertEqual(parsed['oauth_token'], 'uiop')
-        self.assertTrue('oauth_nonce' in parsed)
-        self.assertTrue('oauth_signature' in parsed)
-
-    def test_oauth10a_get_user_coroutine_exception(self):
-        response = self.fetch(
-            '/oauth10a/client/login_coroutine?oauth_token=zxcv&fail_in_get_user=true',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        self.assertEqual(response.code, 503)
-
-    def test_oauth2_redirect(self):
-        response = self.fetch('/oauth2/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue('/oauth2/server/authorize?' in response.headers['Location'])
-
-    def test_facebook_login(self):
-        response = self.fetch('/facebook/client/login', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue('/facebook/server/authorize?' in response.headers['Location'])
-        response = self.fetch('/facebook/client/login?code=1234', follow_redirects=False)
-        self.assertEqual(response.code, 200)
-        user = json_decode(response.body)
-        self.assertEqual(user['access_token'], 'asdf')
-        self.assertEqual(user['session_expires'], '3600')
-
-    def base_twitter_redirect(self, url):
-        # Same as test_oauth10a_redirect
-        response = self.fetch(url, follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/oauth1/server/authorize?oauth_token=zxcv'))
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_twitter_redirect_legacy(self):
-        self.base_twitter_redirect('/legacy/twitter/client/login')
-
-    def test_twitter_redirect(self):
-        self.base_twitter_redirect('/twitter/client/login')
-
-    def test_twitter_redirect_gen_engine(self):
-        self.base_twitter_redirect('/twitter/client/login_gen_engine')
-
-    def test_twitter_redirect_gen_coroutine(self):
-        self.base_twitter_redirect('/twitter/client/login_gen_coroutine')
-
-    def test_twitter_authenticate_redirect(self):
-        response = self.fetch('/twitter/client/authenticate', follow_redirects=False)
-        self.assertEqual(response.code, 302)
-        self.assertTrue(response.headers['Location'].endswith(
-            '/twitter/server/authenticate?oauth_token=zxcv'), response.headers['Location'])
-        # the cookie is base64('zxcv')|base64('1234')
-        self.assertTrue(
-            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
-            response.headers['Set-Cookie'])
-
-    def test_twitter_get_user(self):
-        response = self.fetch(
-            '/twitter/client/login?oauth_token=zxcv',
-            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
-        response.rethrow()
-        parsed = json_decode(response.body)
-        self.assertEqual(parsed,
-                         {u'access_token': {u'key': u'hjkl',
-                                            u'screen_name': u'foo',
-                                            u'secret': u'vbnm'},
-                          u'name': u'Foo',
-                          u'screen_name': u'foo',
-                          u'username': u'foo'})
-
-    def test_twitter_show_user_legacy(self):
-        response = self.fetch('/legacy/twitter/client/show_user?name=somebody')
-        response.rethrow()
-        self.assertEqual(json_decode(response.body),
-                         {'name': 'Somebody', 'screen_name': 'somebody'})
-
-    def test_twitter_show_user_error_legacy(self):
-        with ExpectLog(gen_log, 'Error response HTTP 500'):
-            response = self.fetch('/legacy/twitter/client/show_user?name=error')
-        self.assertEqual(response.code, 500)
-        self.assertEqual(response.body, b'error from twitter request')
-
-    def test_twitter_show_user(self):
-        response = self.fetch('/twitter/client/show_user?name=somebody')
-        response.rethrow()
-        self.assertEqual(json_decode(response.body),
-                         {'name': 'Somebody', 'screen_name': 'somebody'})
-
-    def test_twitter_show_user_error(self):
-        response = self.fetch('/twitter/client/show_user?name=error')
-        self.assertEqual(response.code, 500)
-        self.assertEqual(response.body, b'error from twitter request')
-
-
-class GoogleLoginHandler(RequestHandler, GoogleOAuth2Mixin):
-    def initialize(self, test):
-        self.test = test
-        self._OAUTH_REDIRECT_URI = test.get_url('/client/login')
-        self._OAUTH_AUTHORIZE_URL = test.get_url('/google/oauth2/authorize')
-        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/google/oauth2/token')
-
-    @gen.coroutine
-    def get(self):
-        code = self.get_argument('code', None)
-        if code is not None:
-            # retrieve authenticate google user
-            access = yield self.get_authenticated_user(self._OAUTH_REDIRECT_URI,
-                                                       code)
-            user = yield self.oauth2_request(
-                self.test.get_url("/google/oauth2/userinfo"),
-                access_token=access["access_token"])
-            # return the user and access token as json
-            user["access_token"] = access["access_token"]
-            self.write(user)
-        else:
-            yield self.authorize_redirect(
-                redirect_uri=self._OAUTH_REDIRECT_URI,
-                client_id=self.settings['google_oauth']['key'],
-                client_secret=self.settings['google_oauth']['secret'],
-                scope=['profile', 'email'],
-                response_type='code',
-                extra_params={'prompt': 'select_account'})
-
-
-class GoogleOAuth2AuthorizeHandler(RequestHandler):
-    def get(self):
-        # issue a fake auth code and redirect to redirect_uri
-        code = 'fake-authorization-code'
-        self.redirect(url_concat(self.get_argument('redirect_uri'),
-                                 dict(code=code)))
-
-
-class GoogleOAuth2TokenHandler(RequestHandler):
-    def post(self):
-        assert self.get_argument('code') == 'fake-authorization-code'
-        # issue a fake token
-        self.finish({
-            'access_token': 'fake-access-token',
-            'expires_in': 'never-expires'
-        })
-
-
-class GoogleOAuth2UserinfoHandler(RequestHandler):
-    def get(self):
-        assert self.get_argument('access_token') == 'fake-access-token'
-        # return a fake user
-        self.finish({
-            'name': 'Foo',
-            'email': 'foo@example.com'
-        })
-
-
-class GoogleOAuth2Test(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application(
-            [
-                # test endpoints
-                ('/client/login', GoogleLoginHandler, dict(test=self)),
-
-                # simulated google authorization server endpoints
-                ('/google/oauth2/authorize', GoogleOAuth2AuthorizeHandler),
-                ('/google/oauth2/token', GoogleOAuth2TokenHandler),
-                ('/google/oauth2/userinfo', GoogleOAuth2UserinfoHandler),
-            ],
-            google_oauth={
-                "key": 'fake_google_client_id',
-                "secret": 'fake_google_client_secret'
-            })
-
-    def test_google_login(self):
-        response = self.fetch('/client/login')
-        self.assertDictEqual({
-            u'name': u'Foo',
-            u'email': u'foo@example.com',
-            u'access_token': u'fake-access-token',
-        }, json_decode(response.body))
diff --git a/lib/tornado/test/autoreload_test.py b/lib/tornado/test/autoreload_test.py
deleted file mode 100755
index 5cbdc2ee..00000000
--- a/lib/tornado/test/autoreload_test.py
+++ /dev/null
@@ -1,114 +0,0 @@
-from __future__ import absolute_import, division, print_function
-import os
-import shutil
-import subprocess
-from subprocess import Popen
-import sys
-from tempfile import mkdtemp
-import time
-
-from tornado.test.util import unittest
-
-
-class AutoreloadTest(unittest.TestCase):
-
-    def test_reload_module(self):
-        main = """\
-import os
-import sys
-
-from tornado import autoreload
-
-# This import will fail if path is not set up correctly
-import testapp
-
-print('Starting')
-if 'TESTAPP_STARTED' not in os.environ:
-    os.environ['TESTAPP_STARTED'] = '1'
-    sys.stdout.flush()
-    autoreload._reload()
-"""
-
-        # Create temporary test application
-        path = mkdtemp()
-        self.addCleanup(shutil.rmtree, path)
-        os.mkdir(os.path.join(path, 'testapp'))
-        open(os.path.join(path, 'testapp/__init__.py'), 'w').close()
-        with open(os.path.join(path, 'testapp/__main__.py'), 'w') as f:
-            f.write(main)
-
-        # Make sure the tornado module under test is available to the test
-        # application
-        pythonpath = os.getcwd()
-        if 'PYTHONPATH' in os.environ:
-            pythonpath += os.pathsep + os.environ['PYTHONPATH']
-
-        p = Popen(
-            [sys.executable, '-m', 'testapp'], stdout=subprocess.PIPE,
-            cwd=path, env=dict(os.environ, PYTHONPATH=pythonpath),
-            universal_newlines=True)
-        out = p.communicate()[0]
-        self.assertEqual(out, 'Starting\nStarting\n')
-
-    def test_reload_wrapper_preservation(self):
-        # This test verifies that when `python -m tornado.autoreload`
-        # is used on an application that also has an internal
-        # autoreload, the reload wrapper is preserved on restart.
-        main = """\
-import os
-import sys
-
-# This import will fail if path is not set up correctly
-import testapp
-
-if 'tornado.autoreload' not in sys.modules:
-    raise Exception('started without autoreload wrapper')
-
-import tornado.autoreload
-
-print('Starting')
-sys.stdout.flush()
-if 'TESTAPP_STARTED' not in os.environ:
-    os.environ['TESTAPP_STARTED'] = '1'
-    # Simulate an internal autoreload (one not caused
-    # by the wrapper).
-    tornado.autoreload._reload()
-else:
-    # Exit directly so autoreload doesn't catch it.
-    os._exit(0)
-"""
-
-        # Create temporary test application
-        path = mkdtemp()
-        os.mkdir(os.path.join(path, 'testapp'))
-        self.addCleanup(shutil.rmtree, path)
-        init_file = os.path.join(path, 'testapp', '__init__.py')
-        open(init_file, 'w').close()
-        main_file = os.path.join(path, 'testapp', '__main__.py')
-        with open(main_file, 'w') as f:
-            f.write(main)
-
-        # Make sure the tornado module under test is available to the test
-        # application
-        pythonpath = os.getcwd()
-        if 'PYTHONPATH' in os.environ:
-            pythonpath += os.pathsep + os.environ['PYTHONPATH']
-
-        autoreload_proc = Popen(
-            [sys.executable, '-m', 'tornado.autoreload', '-m', 'testapp'],
-            stdout=subprocess.PIPE, cwd=path,
-            env=dict(os.environ, PYTHONPATH=pythonpath),
-            universal_newlines=True)
-
-        # This timeout needs to be fairly generous for pypy due to jit
-        # warmup costs.
-        for i in range(40):
-            if autoreload_proc.poll() is not None:
-                break
-            time.sleep(0.1)
-        else:
-            autoreload_proc.kill()
-            raise Exception("subprocess failed to terminate")
-
-        out = autoreload_proc.communicate()[0]
-        self.assertEqual(out, 'Starting\n' * 2)
diff --git a/lib/tornado/test/concurrent_test.py b/lib/tornado/test/concurrent_test.py
deleted file mode 100755
index 737c13e1..00000000
--- a/lib/tornado/test/concurrent_test.py
+++ /dev/null
@@ -1,496 +0,0 @@
-#
-# Copyright 2012 Facebook
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from __future__ import absolute_import, division, print_function
-
-import gc
-import logging
-import re
-import socket
-import sys
-import traceback
-import warnings
-
-from tornado.concurrent import (Future, return_future, ReturnValueIgnoredError,
-                                run_on_executor, future_set_result_unless_cancelled)
-from tornado.escape import utf8, to_unicode
-from tornado import gen
-from tornado.ioloop import IOLoop
-from tornado.iostream import IOStream
-from tornado.log import app_log
-from tornado import stack_context
-from tornado.tcpserver import TCPServer
-from tornado.testing import AsyncTestCase, ExpectLog, bind_unused_port, gen_test
-from tornado.test.util import unittest, skipBefore35, exec_test, ignore_deprecation
-
-
-try:
-    from concurrent import futures
-except ImportError:
-    futures = None
-
-
-class MiscFutureTest(AsyncTestCase):
-
-    def test_future_set_result_unless_cancelled(self):
-        fut = Future()
-        future_set_result_unless_cancelled(fut, 42)
-        self.assertEqual(fut.result(), 42)
-        self.assertFalse(fut.cancelled())
-
-        fut = Future()
-        fut.cancel()
-        is_cancelled = fut.cancelled()
-        future_set_result_unless_cancelled(fut, 42)
-        self.assertEqual(fut.cancelled(), is_cancelled)
-        if not is_cancelled:
-            self.assertEqual(fut.result(), 42)
-
-
-class ReturnFutureTest(AsyncTestCase):
-    with ignore_deprecation():
-        @return_future
-        def sync_future(self, callback):
-            callback(42)
-
-        @return_future
-        def async_future(self, callback):
-            self.io_loop.add_callback(callback, 42)
-
-        @return_future
-        def immediate_failure(self, callback):
-            1 / 0
-
-        @return_future
-        def delayed_failure(self, callback):
-            self.io_loop.add_callback(lambda: 1 / 0)
-
-        @return_future
-        def return_value(self, callback):
-            # Note that the result of both running the callback and returning
-            # a value (or raising an exception) is unspecified; with current
-            # implementations the last event prior to callback resolution wins.
-            return 42
-
-        @return_future
-        def no_result_future(self, callback):
-            callback()
-
-    def test_immediate_failure(self):
-        with self.assertRaises(ZeroDivisionError):
-            # The caller sees the error just like a normal function.
-            self.immediate_failure(callback=self.stop)
-        # The callback is not run because the function failed synchronously.
-        self.io_loop.add_timeout(self.io_loop.time() + 0.05, self.stop)
-        result = self.wait()
-        self.assertIs(result, None)
-
-    def test_return_value(self):
-        with self.assertRaises(ReturnValueIgnoredError):
-            self.return_value(callback=self.stop)
-
-    def test_callback_kw(self):
-        with ignore_deprecation():
-            future = self.sync_future(callback=self.stop)
-        result = self.wait()
-        self.assertEqual(result, 42)
-        self.assertEqual(future.result(), 42)
-
-    def test_callback_positional(self):
-        # When the callback is passed in positionally, future_wrap shouldn't
-        # add another callback in the kwargs.
-        with ignore_deprecation():
-            future = self.sync_future(self.stop)
-        result = self.wait()
-        self.assertEqual(result, 42)
-        self.assertEqual(future.result(), 42)
-
-    def test_no_callback(self):
-        future = self.sync_future()
-        self.assertEqual(future.result(), 42)
-
-    def test_none_callback_kw(self):
-        # explicitly pass None as callback
-        future = self.sync_future(callback=None)
-        self.assertEqual(future.result(), 42)
-
-    def test_none_callback_pos(self):
-        future = self.sync_future(None)
-        self.assertEqual(future.result(), 42)
-
-    def test_async_future(self):
-        future = self.async_future()
-        self.assertFalse(future.done())
-        self.io_loop.add_future(future, self.stop)
-        future2 = self.wait()
-        self.assertIs(future, future2)
-        self.assertEqual(future.result(), 42)
-
-    @gen_test
-    def test_async_future_gen(self):
-        result = yield self.async_future()
-        self.assertEqual(result, 42)
-
-    def test_delayed_failure(self):
-        future = self.delayed_failure()
-        with ignore_deprecation():
-            self.io_loop.add_future(future, self.stop)
-            future2 = self.wait()
-        self.assertIs(future, future2)
-        with self.assertRaises(ZeroDivisionError):
-            future.result()
-
-    def test_kw_only_callback(self):
-        with ignore_deprecation():
-            @return_future
-            def f(**kwargs):
-                kwargs['callback'](42)
-        future = f()
-        self.assertEqual(future.result(), 42)
-
-    def test_error_in_callback(self):
-        with ignore_deprecation():
-            self.sync_future(callback=lambda future: 1 / 0)
-        # The exception gets caught by our StackContext and will be re-raised
-        # when we wait.
-        self.assertRaises(ZeroDivisionError, self.wait)
-
-    def test_no_result_future(self):
-        with ignore_deprecation():
-            future = self.no_result_future(self.stop)
-        result = self.wait()
-        self.assertIs(result, None)
-        # result of this future is undefined, but not an error
-        future.result()
-
-    def test_no_result_future_callback(self):
-        with ignore_deprecation():
-            future = self.no_result_future(callback=lambda: self.stop())
-        result = self.wait()
-        self.assertIs(result, None)
-        future.result()
-
-    @gen_test
-    def test_future_traceback_legacy(self):
-        with ignore_deprecation():
-            @return_future
-            @gen.engine
-            def f(callback):
-                yield gen.Task(self.io_loop.add_callback)
-                try:
-                    1 / 0
-                except ZeroDivisionError:
-                    self.expected_frame = traceback.extract_tb(
-                        sys.exc_info()[2], limit=1)[0]
-                    raise
-            try:
-                yield f()
-                self.fail("didn't get expected exception")
-            except ZeroDivisionError:
-                tb = traceback.extract_tb(sys.exc_info()[2])
-                self.assertIn(self.expected_frame, tb)
-
-    @gen_test
-    def test_future_traceback(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            try:
-                1 / 0
-            except ZeroDivisionError:
-                self.expected_frame = traceback.extract_tb(
-                    sys.exc_info()[2], limit=1)[0]
-                raise
-        try:
-            yield f()
-            self.fail("didn't get expected exception")
-        except ZeroDivisionError:
-            tb = traceback.extract_tb(sys.exc_info()[2])
-            self.assertIn(self.expected_frame, tb)
-
-    @gen_test
-    def test_uncaught_exception_log(self):
-        if IOLoop.configured_class().__name__.endswith('AsyncIOLoop'):
-            # Install an exception handler that mirrors our
-            # non-asyncio logging behavior.
-            def exc_handler(loop, context):
-                app_log.error('%s: %s', context['message'],
-                              type(context.get('exception')))
-            self.io_loop.asyncio_loop.set_exception_handler(exc_handler)
-
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            1 / 0
-
-        g = f()
-
-        with ExpectLog(app_log,
-                       "(?s)Future.* exception was never retrieved:"
-                       ".*ZeroDivisionError"):
-            yield gen.moment
-            yield gen.moment
-            # For some reason, TwistedIOLoop and pypy3 need a third iteration
-            # in order to drain references to the future
-            yield gen.moment
-            del g
-            gc.collect()  # for PyPy
-
-
-# The following series of classes demonstrate and test various styles
-# of use, with and without generators and futures.
-
-
-class CapServer(TCPServer):
-    @gen.coroutine
-    def handle_stream(self, stream, address):
-        data = yield stream.read_until(b"\n")
-        data = to_unicode(data)
-        if data == data.upper():
-            stream.write(b"error\talready capitalized\n")
-        else:
-            # data already has \n
-            stream.write(utf8("ok\t%s" % data.upper()))
-        stream.close()
-
-
-class CapError(Exception):
-    pass
-
-
-class BaseCapClient(object):
-    def __init__(self, port):
-        self.port = port
-
-    def process_response(self, data):
-        status, message = re.match('(.*)\t(.*)\n', to_unicode(data)).groups()
-        if status == 'ok':
-            return message
-        else:
-            raise CapError(message)
-
-
-class ManualCapClient(BaseCapClient):
-    def capitalize(self, request_data, callback=None):
-        logging.debug("capitalize")
-        self.request_data = request_data
-        self.stream = IOStream(socket.socket())
-        self.stream.connect(('127.0.0.1', self.port),
-                            callback=self.handle_connect)
-        self.future = Future()
-        if callback is not None:
-            self.future.add_done_callback(
-                stack_context.wrap(lambda future: callback(future.result())))
-        return self.future
-
-    def handle_connect(self):
-        logging.debug("handle_connect")
-        self.stream.write(utf8(self.request_data + "\n"))
-        self.stream.read_until(b'\n', callback=self.handle_read)
-
-    def handle_read(self, data):
-        logging.debug("handle_read")
-        self.stream.close()
-        try:
-            self.future.set_result(self.process_response(data))
-        except CapError as e:
-            self.future.set_exception(e)
-
-
-class DecoratorCapClient(BaseCapClient):
-    with ignore_deprecation():
-        @return_future
-        def capitalize(self, request_data, callback):
-            logging.debug("capitalize")
-            self.request_data = request_data
-            self.stream = IOStream(socket.socket())
-            self.stream.connect(('127.0.0.1', self.port),
-                                callback=self.handle_connect)
-            self.callback = callback
-
-    def handle_connect(self):
-        logging.debug("handle_connect")
-        self.stream.write(utf8(self.request_data + "\n"))
-        self.stream.read_until(b'\n', callback=self.handle_read)
-
-    def handle_read(self, data):
-        logging.debug("handle_read")
-        self.stream.close()
-        self.callback(self.process_response(data))
-
-
-class GeneratorCapClient(BaseCapClient):
-    @gen.coroutine
-    def capitalize(self, request_data):
-        logging.debug('capitalize')
-        stream = IOStream(socket.socket())
-        logging.debug('connecting')
-        yield stream.connect(('127.0.0.1', self.port))
-        stream.write(utf8(request_data + '\n'))
-        logging.debug('reading')
-        data = yield stream.read_until(b'\n')
-        logging.debug('returning')
-        stream.close()
-        raise gen.Return(self.process_response(data))
-
-
-class ClientTestMixin(object):
-    def setUp(self):
-        super(ClientTestMixin, self).setUp()  # type: ignore
-        self.server = CapServer()
-        sock, port = bind_unused_port()
-        self.server.add_sockets([sock])
-        self.client = self.client_class(port=port)
-
-    def tearDown(self):
-        self.server.stop()
-        super(ClientTestMixin, self).tearDown()  # type: ignore
-
-    def test_callback(self):
-        with ignore_deprecation():
-            self.client.capitalize("hello", callback=self.stop)
-        result = self.wait()
-        self.assertEqual(result, "HELLO")
-
-    def test_callback_error(self):
-        with ignore_deprecation():
-            self.client.capitalize("HELLO", callback=self.stop)
-            self.assertRaisesRegexp(CapError, "already capitalized", self.wait)
-
-    def test_future(self):
-        future = self.client.capitalize("hello")
-        self.io_loop.add_future(future, self.stop)
-        self.wait()
-        self.assertEqual(future.result(), "HELLO")
-
-    def test_future_error(self):
-        future = self.client.capitalize("HELLO")
-        self.io_loop.add_future(future, self.stop)
-        self.wait()
-        self.assertRaisesRegexp(CapError, "already capitalized", future.result)
-
-    def test_generator(self):
-        @gen.coroutine
-        def f():
-            result = yield self.client.capitalize("hello")
-            self.assertEqual(result, "HELLO")
-        self.io_loop.run_sync(f)
-
-    def test_generator_error(self):
-        @gen.coroutine
-        def f():
-            with self.assertRaisesRegexp(CapError, "already capitalized"):
-                yield self.client.capitalize("HELLO")
-        self.io_loop.run_sync(f)
-
-
-class ManualClientTest(ClientTestMixin, AsyncTestCase):
-    client_class = ManualCapClient
-
-    def setUp(self):
-        self.warning_catcher = warnings.catch_warnings()
-        self.warning_catcher.__enter__()
-        warnings.simplefilter('ignore', DeprecationWarning)
-        super(ManualClientTest, self).setUp()
-
-    def tearDown(self):
-        super(ManualClientTest, self).tearDown()
-        self.warning_catcher.__exit__(None, None, None)
-
-
-class DecoratorClientTest(ClientTestMixin, AsyncTestCase):
-    client_class = DecoratorCapClient
-
-    def setUp(self):
-        self.warning_catcher = warnings.catch_warnings()
-        self.warning_catcher.__enter__()
-        warnings.simplefilter('ignore', DeprecationWarning)
-        super(DecoratorClientTest, self).setUp()
-
-    def tearDown(self):
-        super(DecoratorClientTest, self).tearDown()
-        self.warning_catcher.__exit__(None, None, None)
-
-
-class GeneratorClientTest(ClientTestMixin, AsyncTestCase):
-    client_class = GeneratorCapClient
-
-
-@unittest.skipIf(futures is None, "concurrent.futures module not present")
-class RunOnExecutorTest(AsyncTestCase):
-    @gen_test
-    def test_no_calling(self):
-        class Object(object):
-            def __init__(self):
-                self.executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor
-            def f(self):
-                return 42
-
-        o = Object()
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @gen_test
-    def test_call_with_no_args(self):
-        class Object(object):
-            def __init__(self):
-                self.executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor()
-            def f(self):
-                return 42
-
-        o = Object()
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @gen_test
-    def test_call_with_executor(self):
-        class Object(object):
-            def __init__(self):
-                self.__executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor(executor='_Object__executor')
-            def f(self):
-                return 42
-
-        o = Object()
-        answer = yield o.f()
-        self.assertEqual(answer, 42)
-
-    @skipBefore35
-    @gen_test
-    def test_async_await(self):
-        class Object(object):
-            def __init__(self):
-                self.executor = futures.thread.ThreadPoolExecutor(1)
-
-            @run_on_executor()
-            def f(self):
-                return 42
-
-        o = Object()
-        namespace = exec_test(globals(), locals(), """
-        async def f():
-            answer = await o.f()
-            return answer
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/lib/tornado/test/csv_translations/fr_FR.csv b/lib/tornado/test/csv_translations/fr_FR.csv
deleted file mode 100755
index 6321b6e7..00000000
--- a/lib/tornado/test/csv_translations/fr_FR.csv
+++ /dev/null
@@ -1 +0,0 @@
-"school","école"
diff --git a/lib/tornado/test/curl_httpclient_test.py b/lib/tornado/test/curl_httpclient_test.py
deleted file mode 100755
index 4230d4cd..00000000
--- a/lib/tornado/test/curl_httpclient_test.py
+++ /dev/null
@@ -1,153 +0,0 @@
-# coding: utf-8
-from __future__ import absolute_import, division, print_function
-
-from hashlib import md5
-
-from tornado.escape import utf8
-from tornado.httpclient import HTTPRequest, HTTPClientError
-from tornado.locks import Event
-from tornado.stack_context import ExceptionStackContext
-from tornado.testing import AsyncHTTPTestCase, gen_test
-from tornado.test import httpclient_test
-from tornado.test.util import unittest, ignore_deprecation
-from tornado.web import Application, RequestHandler
-
-
-try:
-    import pycurl  # type: ignore
-except ImportError:
-    pycurl = None
-
-if pycurl is not None:
-    from tornado.curl_httpclient import CurlAsyncHTTPClient
-
-
-@unittest.skipIf(pycurl is None, "pycurl module not present")
-class CurlHTTPClientCommonTestCase(httpclient_test.HTTPClientCommonTestCase):
-    def get_http_client(self):
-        client = CurlAsyncHTTPClient(defaults=dict(allow_ipv6=False))
-        # make sure AsyncHTTPClient magic doesn't give us the wrong class
-        self.assertTrue(isinstance(client, CurlAsyncHTTPClient))
-        return client
-
-
-class DigestAuthHandler(RequestHandler):
-    def initialize(self, username, password):
-        self.username = username
-        self.password = password
-
-    def get(self):
-        realm = 'test'
-        opaque = 'asdf'
-        # Real implementations would use a random nonce.
-        nonce = "1234"
-
-        auth_header = self.request.headers.get('Authorization', None)
-        if auth_header is not None:
-            auth_mode, params = auth_header.split(' ', 1)
-            assert auth_mode == 'Digest'
-            param_dict = {}
-            for pair in params.split(','):
-                k, v = pair.strip().split('=', 1)
-                if v[0] == '"' and v[-1] == '"':
-                    v = v[1:-1]
-                param_dict[k] = v
-            assert param_dict['realm'] == realm
-            assert param_dict['opaque'] == opaque
-            assert param_dict['nonce'] == nonce
-            assert param_dict['username'] == self.username
-            assert param_dict['uri'] == self.request.path
-            h1 = md5(utf8('%s:%s:%s' % (self.username, realm, self.password))).hexdigest()
-            h2 = md5(utf8('%s:%s' % (self.request.method,
-                                     self.request.path))).hexdigest()
-            digest = md5(utf8('%s:%s:%s' % (h1, nonce, h2))).hexdigest()
-            if digest == param_dict['response']:
-                self.write('ok')
-            else:
-                self.write('fail')
-        else:
-            self.set_status(401)
-            self.set_header('WWW-Authenticate',
-                            'Digest realm="%s", nonce="%s", opaque="%s"' %
-                            (realm, nonce, opaque))
-
-
-class CustomReasonHandler(RequestHandler):
-    def get(self):
-        self.set_status(200, "Custom reason")
-
-
-class CustomFailReasonHandler(RequestHandler):
-    def get(self):
-        self.set_status(400, "Custom reason")
-
-
-@unittest.skipIf(pycurl is None, "pycurl module not present")
-class CurlHTTPClientTestCase(AsyncHTTPTestCase):
-    def setUp(self):
-        super(CurlHTTPClientTestCase, self).setUp()
-        self.http_client = self.create_client()
-
-    def get_app(self):
-        return Application([
-            ('/digest', DigestAuthHandler, {'username': 'foo', 'password': 'bar'}),
-            ('/digest_non_ascii', DigestAuthHandler, {'username': 'foo', 'password': 'barユ£'}),
-            ('/custom_reason', CustomReasonHandler),
-            ('/custom_fail_reason', CustomFailReasonHandler),
-        ])
-
-    def create_client(self, **kwargs):
-        return CurlAsyncHTTPClient(force_instance=True,
-                                   defaults=dict(allow_ipv6=False),
-                                   **kwargs)
-
-    @gen_test
-    def test_prepare_curl_callback_stack_context(self):
-        exc_info = []
-        error_event = Event()
-
-        def error_handler(typ, value, tb):
-            exc_info.append((typ, value, tb))
-            error_event.set()
-            return True
-
-        with ignore_deprecation():
-            with ExceptionStackContext(error_handler):
-                request = HTTPRequest(self.get_url('/custom_reason'),
-                                      prepare_curl_callback=lambda curl: 1 / 0)
-        yield [error_event.wait(), self.http_client.fetch(request)]
-        self.assertEqual(1, len(exc_info))
-        self.assertIs(exc_info[0][0], ZeroDivisionError)
-
-    def test_digest_auth(self):
-        response = self.fetch('/digest', auth_mode='digest',
-                              auth_username='foo', auth_password='bar')
-        self.assertEqual(response.body, b'ok')
-
-    def test_custom_reason(self):
-        response = self.fetch('/custom_reason')
-        self.assertEqual(response.reason, "Custom reason")
-
-    def test_fail_custom_reason(self):
-        response = self.fetch('/custom_fail_reason')
-        self.assertEqual(str(response.error), "HTTP 400: Custom reason")
-
-    def test_failed_setup(self):
-        self.http_client = self.create_client(max_clients=1)
-        for i in range(5):
-            with ignore_deprecation():
-                response = self.fetch(u'/ユニコード')
-            self.assertIsNot(response.error, None)
-
-            with self.assertRaises((UnicodeEncodeError, HTTPClientError)):
-                # This raises UnicodeDecodeError on py3 and
-                # HTTPClientError(404) on py2. The main motivation of
-                # this test is to ensure that the UnicodeEncodeError
-                # during the setup phase doesn't lead the request to
-                # be dropped on the floor.
-                response = self.fetch(u'/ユニコード', raise_error=True)
-
-    def test_digest_auth_non_ascii(self):
-        response = self.fetch('/digest_non_ascii', auth_mode='digest',
-                              auth_username='foo', auth_password='barユ£')
-        self.assertEqual(response.body, b'ok')
diff --git a/lib/tornado/test/escape_test.py b/lib/tornado/test/escape_test.py
deleted file mode 100755
index f2f2902a..00000000
--- a/lib/tornado/test/escape_test.py
+++ /dev/null
@@ -1,250 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-import tornado.escape
-from tornado.escape import (
-    utf8, xhtml_escape, xhtml_unescape, url_escape, url_unescape,
-    to_unicode, json_decode, json_encode, squeeze, recursive_unicode,
-)
-from tornado.util import unicode_type
-from tornado.test.util import unittest
-
-linkify_tests = [
-    # (input, linkify_kwargs, expected_output)
-
-    ("hello http://world.com/!", {},
-     u'hello http://world.com/!'),
-
-    ("hello http://world.com/with?param=true&stuff=yes", {},
-     u'hello http://world.com/with?param=true&stuff=yes'),  # noqa: E501
-
-    # an opened paren followed by many chars killed Gruber's regex
-    ("http://url.com/w(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {},
-     u'http://url.com/w(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'),  # noqa: E501
-
-    # as did too many dots at the end
-    ("http://url.com/withmany.......................................", {},
-     u'http://url.com/withmany.......................................'),  # noqa: E501
-
-    ("http://url.com/withmany((((((((((((((((((((((((((((((((((a)", {},
-     u'http://url.com/withmany((((((((((((((((((((((((((((((((((a)'),  # noqa: E501
-
-    # some examples from http://daringfireball.net/2009/11/liberal_regex_for_matching_urls
-    # plus a fex extras (such as multiple parentheses).
-    ("http://foo.com/blah_blah", {},
-     u'http://foo.com/blah_blah'),
-
-    ("http://foo.com/blah_blah/", {},
-     u'http://foo.com/blah_blah/'),
-
-    ("(Something like http://foo.com/blah_blah)", {},
-     u'(Something like http://foo.com/blah_blah)'),
-
-    ("http://foo.com/blah_blah_(wikipedia)", {},
-     u'http://foo.com/blah_blah_(wikipedia)'),
-
-    ("http://foo.com/blah_(blah)_(wikipedia)_blah", {},
-     u'http://foo.com/blah_(blah)_(wikipedia)_blah'),  # noqa: E501
-
-    ("(Something like http://foo.com/blah_blah_(wikipedia))", {},
-     u'(Something like http://foo.com/blah_blah_(wikipedia))'),  # noqa: E501
-
-    ("http://foo.com/blah_blah.", {},
-     u'http://foo.com/blah_blah.'),
-
-    ("http://foo.com/blah_blah/.", {},
-     u'http://foo.com/blah_blah/.'),
-
-    ("", {},
-     u'<http://foo.com/blah_blah>'),
-
-    ("", {},
-     u'<http://foo.com/blah_blah/>'),
-
-    ("http://foo.com/blah_blah,", {},
-     u'http://foo.com/blah_blah,'),
-
-    ("http://www.example.com/wpstyle/?p=364.", {},
-     u'http://www.example.com/wpstyle/?p=364.'),
-
-    ("rdar://1234",
-     {"permitted_protocols": ["http", "rdar"]},
-     u'rdar://1234'),
-
-    ("rdar:/1234",
-     {"permitted_protocols": ["rdar"]},
-     u'rdar:/1234'),
-
-    ("http://userid:password@example.com:8080", {},
-     u'http://userid:password@example.com:8080'),  # noqa: E501
-
-    ("http://userid@example.com", {},
-     u'http://userid@example.com'),
-
-    ("http://userid@example.com:8080", {},
-     u'http://userid@example.com:8080'),
-
-    ("http://userid:password@example.com", {},
-     u'http://userid:password@example.com'),
-
-    ("message://%3c330e7f8409726r6a4ba78dkf1fd71420c1bf6ff@mail.gmail.com%3e",
-     {"permitted_protocols": ["http", "message"]},
-     u''
-     u'message://%3c330e7f8409726r6a4ba78dkf1fd71420c1bf6ff@mail.gmail.com%3e'),
-
-    (u"http://\u27a1.ws/\u4a39", {},
-     u'http://\u27a1.ws/\u4a39'),
-
-    ("http://example.com", {},
-     u'<tag>http://example.com</tag>'),
-
-    ("Just a www.example.com link.", {},
-     u'Just a www.example.com link.'),
-
-    ("Just a www.example.com link.",
-     {"require_protocol": True},
-     u'Just a www.example.com link.'),
-
-    ("A http://reallylong.com/link/that/exceedsthelenglimit.html",
-     {"require_protocol": True, "shorten": True},
-     u'A http://reallylong.com/link...'),  # noqa: E501
-
-    ("A http://reallylongdomainnamethatwillbetoolong.com/hi!",
-     {"shorten": True},
-     u'A http://reallylongdomainnametha...!'),  # noqa: E501
-
-    ("A file:///passwords.txt and http://web.com link", {},
-     u'A file:///passwords.txt and http://web.com link'),
-
-    ("A file:///passwords.txt and http://web.com link",
-     {"permitted_protocols": ["file"]},
-     u'A file:///passwords.txt and http://web.com link'),
-
-    ("www.external-link.com",
-     {"extra_params": 'rel="nofollow" class="external"'},
-     u'www.external-link.com'),  # noqa: E501
-
-    ("www.external-link.com and www.internal-link.com/blogs extra",
-     {"extra_params": lambda href: 'class="internal"' if href.startswith("http://www.internal-link.com") else 'rel="nofollow" class="external"'},  # noqa: E501
-     u'www.external-link.com'            # noqa: E501
-     u' and www.internal-link.com/blogs extra'),  # noqa: E501
-
-    ("www.external-link.com",
-     {"extra_params": lambda href: '    rel="nofollow" class="external"  '},
-     u'www.external-link.com'),  # noqa: E501
-]
-
-
-class EscapeTestCase(unittest.TestCase):
-    def test_linkify(self):
-        for text, kwargs, html in linkify_tests:
-            linked = tornado.escape.linkify(text, **kwargs)
-            self.assertEqual(linked, html)
-
-    def test_xhtml_escape(self):
-        tests = [
-            ("", "<foo>"),
-            (u"", u"<foo>"),
-            (b"", b"<foo>"),
-
-            ("<>&\"'", "<>&"'"),
-            ("&", "&amp;"),
-
-            (u"<\u00e9>", u"<\u00e9>"),
-            (b"<\xc3\xa9>", b"<\xc3\xa9>"),
-        ]
-        for unescaped, escaped in tests:
-            self.assertEqual(utf8(xhtml_escape(unescaped)), utf8(escaped))
-            self.assertEqual(utf8(unescaped), utf8(xhtml_unescape(escaped)))
-
-    def test_xhtml_unescape_numeric(self):
-        tests = [
-            ('foo bar', 'foo bar'),
-            ('foo bar', 'foo bar'),
-            ('foo bar', 'foo bar'),
-            ('foo઼bar', u'foo\u0abcbar'),
-            ('foo&#xyz;bar', 'foo&#xyz;bar'),  # invalid encoding
-            ('foo&#;bar', 'foo&#;bar'),        # invalid encoding
-            ('foo&#x;bar', 'foo&#x;bar'),      # invalid encoding
-        ]
-        for escaped, unescaped in tests:
-            self.assertEqual(unescaped, xhtml_unescape(escaped))
-
-    def test_url_escape_unicode(self):
-        tests = [
-            # byte strings are passed through as-is
-            (u'\u00e9'.encode('utf8'), '%C3%A9'),
-            (u'\u00e9'.encode('latin1'), '%E9'),
-
-            # unicode strings become utf8
-            (u'\u00e9', '%C3%A9'),
-        ]
-        for unescaped, escaped in tests:
-            self.assertEqual(url_escape(unescaped), escaped)
-
-    def test_url_unescape_unicode(self):
-        tests = [
-            ('%C3%A9', u'\u00e9', 'utf8'),
-            ('%C3%A9', u'\u00c3\u00a9', 'latin1'),
-            ('%C3%A9', utf8(u'\u00e9'), None),
-        ]
-        for escaped, unescaped, encoding in tests:
-            # input strings to url_unescape should only contain ascii
-            # characters, but make sure the function accepts both byte
-            # and unicode strings.
-            self.assertEqual(url_unescape(to_unicode(escaped), encoding), unescaped)
-            self.assertEqual(url_unescape(utf8(escaped), encoding), unescaped)
-
-    def test_url_escape_quote_plus(self):
-        unescaped = '+ #%'
-        plus_escaped = '%2B+%23%25'
-        escaped = '%2B%20%23%25'
-        self.assertEqual(url_escape(unescaped), plus_escaped)
-        self.assertEqual(url_escape(unescaped, plus=False), escaped)
-        self.assertEqual(url_unescape(plus_escaped), unescaped)
-        self.assertEqual(url_unescape(escaped, plus=False), unescaped)
-        self.assertEqual(url_unescape(plus_escaped, encoding=None),
-                         utf8(unescaped))
-        self.assertEqual(url_unescape(escaped, encoding=None, plus=False),
-                         utf8(unescaped))
-
-    def test_escape_return_types(self):
-        # On python2 the escape methods should generally return the same
-        # type as their argument
-        self.assertEqual(type(xhtml_escape("foo")), str)
-        self.assertEqual(type(xhtml_escape(u"foo")), unicode_type)
-
-    def test_json_decode(self):
-        # json_decode accepts both bytes and unicode, but strings it returns
-        # are always unicode.
-        self.assertEqual(json_decode(b'"foo"'), u"foo")
-        self.assertEqual(json_decode(u'"foo"'), u"foo")
-
-        # Non-ascii bytes are interpreted as utf8
-        self.assertEqual(json_decode(utf8(u'"\u00e9"')), u"\u00e9")
-
-    def test_json_encode(self):
-        # json deals with strings, not bytes.  On python 2 byte strings will
-        # convert automatically if they are utf8; on python 3 byte strings
-        # are not allowed.
-        self.assertEqual(json_decode(json_encode(u"\u00e9")), u"\u00e9")
-        if bytes is str:
-            self.assertEqual(json_decode(json_encode(utf8(u"\u00e9"))), u"\u00e9")
-            self.assertRaises(UnicodeDecodeError, json_encode, b"\xe9")
-
-    def test_squeeze(self):
-        self.assertEqual(squeeze(u'sequences     of    whitespace   chars'),
-                         u'sequences of whitespace chars')
-
-    def test_recursive_unicode(self):
-        tests = {
-            'dict': {b"foo": b"bar"},
-            'list': [b"foo", b"bar"],
-            'tuple': (b"foo", b"bar"),
-            'bytes': b"foo"
-        }
-        self.assertEqual(recursive_unicode(tests['dict']), {u"foo": u"bar"})
-        self.assertEqual(recursive_unicode(tests['list']), [u"foo", u"bar"])
-        self.assertEqual(recursive_unicode(tests['tuple']), (u"foo", u"bar"))
-        self.assertEqual(recursive_unicode(tests['bytes']), u"foo")
diff --git a/lib/tornado/test/gen_test.py b/lib/tornado/test/gen_test.py
deleted file mode 100755
index 346968cf..00000000
--- a/lib/tornado/test/gen_test.py
+++ /dev/null
@@ -1,1862 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-import gc
-import contextlib
-import datetime
-import functools
-import platform
-import sys
-import textwrap
-import time
-import weakref
-import warnings
-
-from tornado.concurrent import return_future, Future
-from tornado.escape import url_escape
-from tornado.httpclient import AsyncHTTPClient
-from tornado.ioloop import IOLoop
-from tornado.log import app_log
-from tornado import stack_context
-from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, ExpectLog, gen_test
-from tornado.test.util import unittest, skipOnTravis, skipBefore33, skipBefore35, skipNotCPython, exec_test, ignore_deprecation  # noqa: E501
-from tornado.web import Application, RequestHandler, asynchronous, HTTPError
-
-from tornado import gen
-
-try:
-    from concurrent import futures
-except ImportError:
-    futures = None
-
-try:
-    import asyncio
-except ImportError:
-    asyncio = None
-
-
-class GenEngineTest(AsyncTestCase):
-    def setUp(self):
-        self.warning_catcher = warnings.catch_warnings()
-        self.warning_catcher.__enter__()
-        warnings.simplefilter('ignore', DeprecationWarning)
-        super(GenEngineTest, self).setUp()
-        self.named_contexts = []
-
-    def tearDown(self):
-        super(GenEngineTest, self).tearDown()
-        self.warning_catcher.__exit__(None, None, None)
-
-    def named_context(self, name):
-        @contextlib.contextmanager
-        def context():
-            self.named_contexts.append(name)
-            try:
-                yield
-            finally:
-                self.assertEqual(self.named_contexts.pop(), name)
-        return context
-
-    def run_gen(self, f):
-        f()
-        return self.wait()
-
-    def delay_callback(self, iterations, callback, arg):
-        """Runs callback(arg) after a number of IOLoop iterations."""
-        if iterations == 0:
-            callback(arg)
-        else:
-            self.io_loop.add_callback(functools.partial(
-                self.delay_callback, iterations - 1, callback, arg))
-
-    with ignore_deprecation():
-        @return_future
-        def async_future(self, result, callback):
-            self.io_loop.add_callback(callback, result)
-
-    @gen.coroutine
-    def async_exception(self, e):
-        yield gen.moment
-        raise e
-
-    def test_no_yield(self):
-        @gen.engine
-        def f():
-            self.stop()
-        self.run_gen(f)
-
-    def test_inline_cb(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))()
-            res = yield gen.Wait("k1")
-            self.assertTrue(res is None)
-            self.stop()
-        self.run_gen(f)
-
-    def test_ioloop_cb(self):
-        @gen.engine
-        def f():
-            self.io_loop.add_callback((yield gen.Callback("k1")))
-            yield gen.Wait("k1")
-            self.stop()
-        self.run_gen(f)
-
-    def test_exception_phase1(self):
-        @gen.engine
-        def f():
-            1 / 0
-        self.assertRaises(ZeroDivisionError, self.run_gen, f)
-
-    def test_exception_phase2(self):
-        @gen.engine
-        def f():
-            self.io_loop.add_callback((yield gen.Callback("k1")))
-            yield gen.Wait("k1")
-            1 / 0
-        self.assertRaises(ZeroDivisionError, self.run_gen, f)
-
-    def test_exception_in_task_phase1(self):
-        def fail_task(callback):
-            1 / 0
-
-        @gen.engine
-        def f():
-            try:
-                yield gen.Task(fail_task)
-                raise Exception("did not get expected exception")
-            except ZeroDivisionError:
-                self.stop()
-        self.run_gen(f)
-
-    def test_exception_in_task_phase2(self):
-        # This is the case that requires the use of stack_context in gen.engine
-        def fail_task(callback):
-            self.io_loop.add_callback(lambda: 1 / 0)
-
-        @gen.engine
-        def f():
-            try:
-                yield gen.Task(fail_task)
-                raise Exception("did not get expected exception")
-            except ZeroDivisionError:
-                self.stop()
-        self.run_gen(f)
-
-    def test_with_arg(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))(42)
-            res = yield gen.Wait("k1")
-            self.assertEqual(42, res)
-            self.stop()
-        self.run_gen(f)
-
-    def test_with_arg_tuple(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback((1, 2)))((3, 4))
-            res = yield gen.Wait((1, 2))
-            self.assertEqual((3, 4), res)
-            self.stop()
-        self.run_gen(f)
-
-    def test_key_reuse(self):
-        @gen.engine
-        def f():
-            yield gen.Callback("k1")
-            yield gen.Callback("k1")
-            self.stop()
-        self.assertRaises(gen.KeyReuseError, self.run_gen, f)
-
-    def test_key_reuse_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Callback((1, 2))
-            yield gen.Callback((1, 2))
-            self.stop()
-        self.assertRaises(gen.KeyReuseError, self.run_gen, f)
-
-    def test_key_mismatch(self):
-        @gen.engine
-        def f():
-            yield gen.Callback("k1")
-            yield gen.Wait("k2")
-            self.stop()
-        self.assertRaises(gen.UnknownKeyError, self.run_gen, f)
-
-    def test_key_mismatch_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Callback((1, 2))
-            yield gen.Wait((2, 3))
-            self.stop()
-        self.assertRaises(gen.UnknownKeyError, self.run_gen, f)
-
-    def test_leaked_callback(self):
-        @gen.engine
-        def f():
-            yield gen.Callback("k1")
-            self.stop()
-        self.assertRaises(gen.LeakedCallbackError, self.run_gen, f)
-
-    def test_leaked_callback_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Callback((1, 2))
-            self.stop()
-        self.assertRaises(gen.LeakedCallbackError, self.run_gen, f)
-
-    def test_parallel_callback(self):
-        @gen.engine
-        def f():
-            for k in range(3):
-                self.io_loop.add_callback((yield gen.Callback(k)))
-            yield gen.Wait(1)
-            self.io_loop.add_callback((yield gen.Callback(3)))
-            yield gen.Wait(0)
-            yield gen.Wait(3)
-            yield gen.Wait(2)
-            self.stop()
-        self.run_gen(f)
-
-    def test_bogus_yield(self):
-        @gen.engine
-        def f():
-            yield 42
-        self.assertRaises(gen.BadYieldError, self.run_gen, f)
-
-    def test_bogus_yield_tuple(self):
-        @gen.engine
-        def f():
-            yield (1, 2)
-        self.assertRaises(gen.BadYieldError, self.run_gen, f)
-
-    def test_reuse(self):
-        @gen.engine
-        def f():
-            self.io_loop.add_callback((yield gen.Callback(0)))
-            yield gen.Wait(0)
-            self.stop()
-        self.run_gen(f)
-        self.run_gen(f)
-
-    def test_task(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            self.stop()
-        self.run_gen(f)
-
-    def test_wait_all(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))("v1")
-            (yield gen.Callback("k2"))("v2")
-            results = yield gen.WaitAll(["k1", "k2"])
-            self.assertEqual(results, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_exception_in_yield(self):
-        @gen.engine
-        def f():
-            try:
-                yield gen.Wait("k1")
-                raise Exception("did not get expected exception")
-            except gen.UnknownKeyError:
-                pass
-            self.stop()
-        self.run_gen(f)
-
-    def test_resume_after_exception_in_yield(self):
-        @gen.engine
-        def f():
-            try:
-                yield gen.Wait("k1")
-                raise Exception("did not get expected exception")
-            except gen.UnknownKeyError:
-                pass
-            (yield gen.Callback("k2"))("v2")
-            self.assertEqual((yield gen.Wait("k2")), "v2")
-            self.stop()
-        self.run_gen(f)
-
-    def test_orphaned_callback(self):
-        @gen.engine
-        def f():
-            self.orphaned_callback = yield gen.Callback(1)
-        try:
-            self.run_gen(f)
-            raise Exception("did not get expected exception")
-        except gen.LeakedCallbackError:
-            pass
-        self.orphaned_callback()
-
-    def test_none(self):
-        @gen.engine
-        def f():
-            yield None
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))("v1")
-            (yield gen.Callback("k2"))("v2")
-            results = yield [gen.Wait("k1"), gen.Wait("k2")]
-            self.assertEqual(results, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_dict(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("k1"))("v1")
-            (yield gen.Callback("k2"))("v2")
-            results = yield dict(foo=gen.Wait("k1"), bar=gen.Wait("k2"))
-            self.assertEqual(results, dict(foo="v1", bar="v2"))
-            self.stop()
-        self.run_gen(f)
-
-    # The following tests explicitly run with both gen.Multi
-    # and gen.multi_future (Task returns a Future, so it can be used
-    # with either).
-    def test_multi_yieldpoint_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.Multi([
-                gen.Task(self.delay_callback, 3, arg="v1"),
-                gen.Task(self.delay_callback, 1, arg="v2"),
-            ])
-            self.assertEqual(responses, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_yieldpoint_dict_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.Multi(dict(
-                foo=gen.Task(self.delay_callback, 3, arg="v1"),
-                bar=gen.Task(self.delay_callback, 1, arg="v2"),
-            ))
-            self.assertEqual(responses, dict(foo="v1", bar="v2"))
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_future_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.multi_future([
-                gen.Task(self.delay_callback, 3, arg="v1"),
-                gen.Task(self.delay_callback, 1, arg="v2"),
-            ])
-            self.assertEqual(responses, ["v1", "v2"])
-            self.stop()
-        self.run_gen(f)
-
-    def test_multi_future_dict_delayed(self):
-        @gen.engine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.multi_future(dict(
-                foo=gen.Task(self.delay_callback, 3, arg="v1"),
-                bar=gen.Task(self.delay_callback, 1, arg="v2"),
-            ))
-            self.assertEqual(responses, dict(foo="v1", bar="v2"))
-            self.stop()
-        self.run_gen(f)
-
-    @skipOnTravis
-    @gen_test
-    def test_multi_performance(self):
-        # Yielding a list used to have quadratic performance; make
-        # sure a large list stays reasonable.  On my laptop a list of
-        # 2000 used to take 1.8s, now it takes 0.12.
-        start = time.time()
-        yield [gen.Task(self.io_loop.add_callback) for i in range(2000)]
-        end = time.time()
-        self.assertLess(end - start, 1.0)
-
-    @gen_test
-    def test_multi_empty(self):
-        # Empty lists or dicts should return the same type.
-        x = yield []
-        self.assertTrue(isinstance(x, list))
-        y = yield {}
-        self.assertTrue(isinstance(y, dict))
-
-    @gen_test
-    def test_multi_mixed_types(self):
-        # A YieldPoint (Wait) and Future (Task) can be combined
-        # (and use the YieldPoint codepath)
-        (yield gen.Callback("k1"))("v1")
-        responses = yield [gen.Wait("k1"),
-                           gen.Task(self.delay_callback, 3, arg="v2")]
-        self.assertEqual(responses, ["v1", "v2"])
-
-    @gen_test
-    def test_future(self):
-        result = yield self.async_future(1)
-        self.assertEqual(result, 1)
-
-    @gen_test
-    def test_multi_future(self):
-        results = yield [self.async_future(1), self.async_future(2)]
-        self.assertEqual(results, [1, 2])
-
-    @gen_test
-    def test_multi_future_duplicate(self):
-        f = self.async_future(2)
-        results = yield [self.async_future(1), f, self.async_future(3), f]
-        self.assertEqual(results, [1, 2, 3, 2])
-
-    @gen_test
-    def test_multi_dict_future(self):
-        results = yield dict(foo=self.async_future(1), bar=self.async_future(2))
-        self.assertEqual(results, dict(foo=1, bar=2))
-
-    @gen_test
-    def test_multi_exceptions(self):
-        with ExpectLog(app_log, "Multiple exceptions in yield list"):
-            with self.assertRaises(RuntimeError) as cm:
-                yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                                 self.async_exception(RuntimeError("error 2"))])
-        self.assertEqual(str(cm.exception), "error 1")
-
-        # With only one exception, no error is logged.
-        with self.assertRaises(RuntimeError):
-            yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                             self.async_future(2)])
-
-        # Exception logging may be explicitly quieted.
-        with self.assertRaises(RuntimeError):
-            yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                             self.async_exception(RuntimeError("error 2"))],
-                            quiet_exceptions=RuntimeError)
-
-    @gen_test
-    def test_multi_future_exceptions(self):
-        with ExpectLog(app_log, "Multiple exceptions in yield list"):
-            with self.assertRaises(RuntimeError) as cm:
-                yield [self.async_exception(RuntimeError("error 1")),
-                       self.async_exception(RuntimeError("error 2"))]
-        self.assertEqual(str(cm.exception), "error 1")
-
-        # With only one exception, no error is logged.
-        with self.assertRaises(RuntimeError):
-            yield [self.async_exception(RuntimeError("error 1")),
-                   self.async_future(2)]
-
-        # Exception logging may be explicitly quieted.
-        with self.assertRaises(RuntimeError):
-            yield gen.multi_future(
-                [self.async_exception(RuntimeError("error 1")),
-                 self.async_exception(RuntimeError("error 2"))],
-                quiet_exceptions=RuntimeError)
-
-    def test_arguments(self):
-        @gen.engine
-        def f():
-            (yield gen.Callback("noargs"))()
-            self.assertEqual((yield gen.Wait("noargs")), None)
-            (yield gen.Callback("1arg"))(42)
-            self.assertEqual((yield gen.Wait("1arg")), 42)
-
-            (yield gen.Callback("kwargs"))(value=42)
-            result = yield gen.Wait("kwargs")
-            self.assertTrue(isinstance(result, gen.Arguments))
-            self.assertEqual(((), dict(value=42)), result)
-            self.assertEqual(dict(value=42), result.kwargs)
-
-            (yield gen.Callback("2args"))(42, 43)
-            result = yield gen.Wait("2args")
-            self.assertTrue(isinstance(result, gen.Arguments))
-            self.assertEqual(((42, 43), {}), result)
-            self.assertEqual((42, 43), result.args)
-
-            def task_func(callback):
-                callback(None, error="foo")
-            result = yield gen.Task(task_func)
-            self.assertTrue(isinstance(result, gen.Arguments))
-            self.assertEqual(((None,), dict(error="foo")), result)
-
-            self.stop()
-        self.run_gen(f)
-
-    def test_stack_context_leak(self):
-        # regression test: repeated invocations of a gen-based
-        # function should not result in accumulated stack_contexts
-        def _stack_depth():
-            head = stack_context._state.contexts[1]
-            length = 0
-
-            while head is not None:
-                length += 1
-                head = head.old_contexts[1]
-
-            return length
-
-        @gen.engine
-        def inner(callback):
-            yield gen.Task(self.io_loop.add_callback)
-            callback()
-
-        @gen.engine
-        def outer():
-            for i in range(10):
-                yield gen.Task(inner)
-
-            stack_increase = _stack_depth() - initial_stack_depth
-            self.assertTrue(stack_increase <= 2)
-            self.stop()
-        initial_stack_depth = _stack_depth()
-        self.run_gen(outer)
-
-    def test_stack_context_leak_exception(self):
-        # same as previous, but with a function that exits with an exception
-        @gen.engine
-        def inner(callback):
-            yield gen.Task(self.io_loop.add_callback)
-            1 / 0
-
-        @gen.engine
-        def outer():
-            for i in range(10):
-                try:
-                    yield gen.Task(inner)
-                except ZeroDivisionError:
-                    pass
-            stack_increase = len(stack_context._state.contexts) - initial_stack_depth
-            self.assertTrue(stack_increase <= 2)
-            self.stop()
-        initial_stack_depth = len(stack_context._state.contexts)
-        self.run_gen(outer)
-
-    def function_with_stack_context(self, callback):
-        # Technically this function should stack_context.wrap its callback
-        # upon entry.  However, it is very common for this step to be
-        # omitted.
-        def step2():
-            self.assertEqual(self.named_contexts, ['a'])
-            self.io_loop.add_callback(callback)
-
-        with stack_context.StackContext(self.named_context('a')):
-            self.io_loop.add_callback(step2)
-
-    @gen_test
-    def test_wait_transfer_stack_context(self):
-        # Wait should not pick up contexts from where callback was invoked,
-        # even if that function improperly fails to wrap its callback.
-        cb = yield gen.Callback('k1')
-        self.function_with_stack_context(cb)
-        self.assertEqual(self.named_contexts, [])
-        yield gen.Wait('k1')
-        self.assertEqual(self.named_contexts, [])
-
-    @gen_test
-    def test_task_transfer_stack_context(self):
-        yield gen.Task(self.function_with_stack_context)
-        self.assertEqual(self.named_contexts, [])
-
-    def test_raise_after_stop(self):
-        # This pattern will be used in the following tests so make sure
-        # the exception propagates as expected.
-        @gen.engine
-        def f():
-            self.stop()
-            1 / 0
-
-        with self.assertRaises(ZeroDivisionError):
-            self.run_gen(f)
-
-    def test_sync_raise_return(self):
-        # gen.Return is allowed in @gen.engine, but it may not be used
-        # to return a value.
-        @gen.engine
-        def f():
-            self.stop(42)
-            raise gen.Return()
-
-        result = self.run_gen(f)
-        self.assertEqual(result, 42)
-
-    def test_async_raise_return(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            self.stop(42)
-            raise gen.Return()
-
-        result = self.run_gen(f)
-        self.assertEqual(result, 42)
-
-    def test_sync_raise_return_value(self):
-        @gen.engine
-        def f():
-            raise gen.Return(42)
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_sync_raise_return_value_tuple(self):
-        @gen.engine
-        def f():
-            raise gen.Return((1, 2))
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_async_raise_return_value(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(42)
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_async_raise_return_value_tuple(self):
-        @gen.engine
-        def f():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return((1, 2))
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_return_value(self):
-        # It is an error to apply @gen.engine to a function that returns
-        # a value.
-        @gen.engine
-        def f():
-            return 42
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    def test_return_value_tuple(self):
-        # It is an error to apply @gen.engine to a function that returns
-        # a value.
-        @gen.engine
-        def f():
-            return (1, 2)
-
-        with self.assertRaises(gen.ReturnValueIgnoredError):
-            self.run_gen(f)
-
-    @skipNotCPython
-    def test_task_refcounting(self):
-        # On CPython, tasks and their arguments should be released immediately
-        # without waiting for garbage collection.
-        @gen.engine
-        def f():
-            class Foo(object):
-                pass
-            arg = Foo()
-            self.arg_ref = weakref.ref(arg)
-            task = gen.Task(self.io_loop.add_callback, arg=arg)
-            self.task_ref = weakref.ref(task)
-            yield task
-            self.stop()
-
-        self.run_gen(f)
-        self.assertIs(self.arg_ref(), None)
-        self.assertIs(self.task_ref(), None)
-
-
-# GenBasicTest duplicates the non-deprecated portions of GenEngineTest
-# with gen.coroutine to ensure we don't lose coverage when gen.engine
-# goes away.
-class GenBasicTest(AsyncTestCase):
-    @gen.coroutine
-    def delay(self, iterations, arg):
-        """Returns arg after a number of IOLoop iterations."""
-        for i in range(iterations):
-            yield gen.moment
-        raise gen.Return(arg)
-
-    with ignore_deprecation():
-        @return_future
-        def async_future(self, result, callback):
-            self.io_loop.add_callback(callback, result)
-
-    @gen.coroutine
-    def async_exception(self, e):
-        yield gen.moment
-        raise e
-
-    @gen.coroutine
-    def add_one_async(self, x):
-        yield gen.moment
-        raise gen.Return(x + 1)
-
-    def test_no_yield(self):
-        @gen.coroutine
-        def f():
-            pass
-        self.io_loop.run_sync(f)
-
-    def test_exception_phase1(self):
-        @gen.coroutine
-        def f():
-            1 / 0
-        self.assertRaises(ZeroDivisionError, self.io_loop.run_sync, f)
-
-    def test_exception_phase2(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            1 / 0
-        self.assertRaises(ZeroDivisionError, self.io_loop.run_sync, f)
-
-    def test_bogus_yield(self):
-        @gen.coroutine
-        def f():
-            yield 42
-        self.assertRaises(gen.BadYieldError, self.io_loop.run_sync, f)
-
-    def test_bogus_yield_tuple(self):
-        @gen.coroutine
-        def f():
-            yield (1, 2)
-        self.assertRaises(gen.BadYieldError, self.io_loop.run_sync, f)
-
-    def test_reuse(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-        self.io_loop.run_sync(f)
-        self.io_loop.run_sync(f)
-
-    def test_none(self):
-        @gen.coroutine
-        def f():
-            yield None
-        self.io_loop.run_sync(f)
-
-    def test_multi(self):
-        @gen.coroutine
-        def f():
-            results = yield [self.add_one_async(1), self.add_one_async(2)]
-            self.assertEqual(results, [2, 3])
-        self.io_loop.run_sync(f)
-
-    def test_multi_dict(self):
-        @gen.coroutine
-        def f():
-            results = yield dict(foo=self.add_one_async(1), bar=self.add_one_async(2))
-            self.assertEqual(results, dict(foo=2, bar=3))
-        self.io_loop.run_sync(f)
-
-    def test_multi_delayed(self):
-        @gen.coroutine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.multi_future([
-                self.delay(3, "v1"),
-                self.delay(1, "v2"),
-            ])
-            self.assertEqual(responses, ["v1", "v2"])
-        self.io_loop.run_sync(f)
-
-    def test_multi_dict_delayed(self):
-        @gen.coroutine
-        def f():
-            # callbacks run at different times
-            responses = yield gen.multi_future(dict(
-                foo=self.delay(3, "v1"),
-                bar=self.delay(1, "v2"),
-            ))
-            self.assertEqual(responses, dict(foo="v1", bar="v2"))
-        self.io_loop.run_sync(f)
-
-    @skipOnTravis
-    @gen_test
-    def test_multi_performance(self):
-        # Yielding a list used to have quadratic performance; make
-        # sure a large list stays reasonable.  On my laptop a list of
-        # 2000 used to take 1.8s, now it takes 0.12.
-        start = time.time()
-        yield [gen.moment for i in range(2000)]
-        end = time.time()
-        self.assertLess(end - start, 1.0)
-
-    @gen_test
-    def test_multi_empty(self):
-        # Empty lists or dicts should return the same type.
-        x = yield []
-        self.assertTrue(isinstance(x, list))
-        y = yield {}
-        self.assertTrue(isinstance(y, dict))
-
-    @gen_test
-    def test_future(self):
-        result = yield self.async_future(1)
-        self.assertEqual(result, 1)
-
-    @gen_test
-    def test_multi_future(self):
-        results = yield [self.async_future(1), self.async_future(2)]
-        self.assertEqual(results, [1, 2])
-
-    @gen_test
-    def test_multi_future_duplicate(self):
-        f = self.async_future(2)
-        results = yield [self.async_future(1), f, self.async_future(3), f]
-        self.assertEqual(results, [1, 2, 3, 2])
-
-    @gen_test
-    def test_multi_dict_future(self):
-        results = yield dict(foo=self.async_future(1), bar=self.async_future(2))
-        self.assertEqual(results, dict(foo=1, bar=2))
-
-    @gen_test
-    def test_multi_exceptions(self):
-        with ExpectLog(app_log, "Multiple exceptions in yield list"):
-            with self.assertRaises(RuntimeError) as cm:
-                yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                                 self.async_exception(RuntimeError("error 2"))])
-        self.assertEqual(str(cm.exception), "error 1")
-
-        # With only one exception, no error is logged.
-        with self.assertRaises(RuntimeError):
-            yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                             self.async_future(2)])
-
-        # Exception logging may be explicitly quieted.
-        with self.assertRaises(RuntimeError):
-            yield gen.Multi([self.async_exception(RuntimeError("error 1")),
-                             self.async_exception(RuntimeError("error 2"))],
-                            quiet_exceptions=RuntimeError)
-
-    @gen_test
-    def test_multi_future_exceptions(self):
-        with ExpectLog(app_log, "Multiple exceptions in yield list"):
-            with self.assertRaises(RuntimeError) as cm:
-                yield [self.async_exception(RuntimeError("error 1")),
-                       self.async_exception(RuntimeError("error 2"))]
-        self.assertEqual(str(cm.exception), "error 1")
-
-        # With only one exception, no error is logged.
-        with self.assertRaises(RuntimeError):
-            yield [self.async_exception(RuntimeError("error 1")),
-                   self.async_future(2)]
-
-        # Exception logging may be explicitly quieted.
-        with self.assertRaises(RuntimeError):
-            yield gen.multi_future(
-                [self.async_exception(RuntimeError("error 1")),
-                 self.async_exception(RuntimeError("error 2"))],
-                quiet_exceptions=RuntimeError)
-
-    def test_sync_raise_return(self):
-        @gen.coroutine
-        def f():
-            raise gen.Return()
-
-        self.io_loop.run_sync(f)
-
-    def test_async_raise_return(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            raise gen.Return()
-
-        self.io_loop.run_sync(f)
-
-    def test_sync_raise_return_value(self):
-        @gen.coroutine
-        def f():
-            raise gen.Return(42)
-
-        self.assertEqual(42, self.io_loop.run_sync(f))
-
-    def test_sync_raise_return_value_tuple(self):
-        @gen.coroutine
-        def f():
-            raise gen.Return((1, 2))
-
-        self.assertEqual((1, 2), self.io_loop.run_sync(f))
-
-    def test_async_raise_return_value(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            raise gen.Return(42)
-
-        self.assertEqual(42, self.io_loop.run_sync(f))
-
-    def test_async_raise_return_value_tuple(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            raise gen.Return((1, 2))
-
-        self.assertEqual((1, 2), self.io_loop.run_sync(f))
-
-
-class GenCoroutineTest(AsyncTestCase):
-    def setUp(self):
-        # Stray StopIteration exceptions can lead to tests exiting prematurely,
-        # so we need explicit checks here to make sure the tests run all
-        # the way through.
-        self.finished = False
-        super(GenCoroutineTest, self).setUp()
-
-    def tearDown(self):
-        super(GenCoroutineTest, self).tearDown()
-        assert self.finished
-
-    def test_attributes(self):
-        self.finished = True
-
-        def f():
-            yield gen.moment
-
-        coro = gen.coroutine(f)
-        self.assertEqual(coro.__name__, f.__name__)
-        self.assertEqual(coro.__module__, f.__module__)
-        self.assertIs(coro.__wrapped__, f)
-
-    def test_is_coroutine_function(self):
-        self.finished = True
-
-        def f():
-            yield gen.moment
-
-        coro = gen.coroutine(f)
-        self.assertFalse(gen.is_coroutine_function(f))
-        self.assertTrue(gen.is_coroutine_function(coro))
-        self.assertFalse(gen.is_coroutine_function(coro()))
-
-    @gen_test
-    def test_sync_gen_return(self):
-        @gen.coroutine
-        def f():
-            raise gen.Return(42)
-        result = yield f()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_async_gen_return(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            raise gen.Return(42)
-        result = yield f()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_sync_return(self):
-        @gen.coroutine
-        def f():
-            return 42
-        result = yield f()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore33
-    @gen_test
-    def test_async_return(self):
-        namespace = exec_test(globals(), locals(), """
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            return 42
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore33
-    @gen_test
-    def test_async_early_return(self):
-        # A yield statement exists but is not executed, which means
-        # this function "returns" via an exception.  This exception
-        # doesn't happen before the exception handling is set up.
-        namespace = exec_test(globals(), locals(), """
-        @gen.coroutine
-        def f():
-            if True:
-                return 42
-            yield gen.Task(self.io_loop.add_callback)
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_await(self):
-        @gen.coroutine
-        def f1():
-            yield gen.moment
-            raise gen.Return(42)
-
-        # This test verifies that an async function can await a
-        # yield-based gen.coroutine, and that a gen.coroutine
-        # (the test method itself) can yield an async function.
-        namespace = exec_test(globals(), locals(), """
-        async def f2():
-            result = await f1()
-            return result
-        """)
-        result = yield namespace['f2']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_asyncio_sleep_zero(self):
-        # asyncio.sleep(0) turns into a special case (equivalent to
-        # `yield None`)
-        namespace = exec_test(globals(), locals(), """
-        async def f():
-            import asyncio
-            await asyncio.sleep(0)
-            return 42
-        """)
-        result = yield namespace['f']()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_await_mixed_multi_native_future(self):
-        @gen.coroutine
-        def f1():
-            yield gen.moment
-
-        namespace = exec_test(globals(), locals(), """
-        async def f2():
-            await f1()
-            return 42
-        """)
-
-        @gen.coroutine
-        def f3():
-            yield gen.moment
-            raise gen.Return(43)
-
-        results = yield [namespace['f2'](), f3()]
-        self.assertEqual(results, [42, 43])
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_await_mixed_multi_native_yieldpoint(self):
-        namespace = exec_test(globals(), locals(), """
-        async def f1():
-            await gen.Task(self.io_loop.add_callback)
-            return 42
-        """)
-
-        @gen.coroutine
-        def f2():
-            yield gen.Task(self.io_loop.add_callback)
-            raise gen.Return(43)
-
-        with ignore_deprecation():
-            f2(callback=(yield gen.Callback('cb')))
-            results = yield [namespace['f1'](), gen.Wait('cb')]
-        self.assertEqual(results, [42, 43])
-        self.finished = True
-
-    @skipBefore35
-    @gen_test
-    def test_async_with_timeout(self):
-        namespace = exec_test(globals(), locals(), """
-        async def f1():
-            return 42
-        """)
-
-        result = yield gen.with_timeout(datetime.timedelta(hours=1),
-                                        namespace['f1']())
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_sync_return_no_value(self):
-        @gen.coroutine
-        def f():
-            return
-        result = yield f()
-        self.assertEqual(result, None)
-        self.finished = True
-
-    @gen_test
-    def test_async_return_no_value(self):
-        # Without a return value we don't need python 3.3.
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            return
-        result = yield f()
-        self.assertEqual(result, None)
-        self.finished = True
-
-    @gen_test
-    def test_sync_raise(self):
-        @gen.coroutine
-        def f():
-            1 / 0
-        # The exception is raised when the future is yielded
-        # (or equivalently when its result method is called),
-        # not when the function itself is called).
-        future = f()
-        with self.assertRaises(ZeroDivisionError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_async_raise(self):
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            1 / 0
-        future = f()
-        with self.assertRaises(ZeroDivisionError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_pass_callback(self):
-        with ignore_deprecation():
-            @gen.coroutine
-            def f():
-                raise gen.Return(42)
-            result = yield gen.Task(f)
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_replace_yieldpoint_exception(self):
-        # Test exception handling: a coroutine can catch one exception
-        # raised by a yield point and raise a different one.
-        @gen.coroutine
-        def f1():
-            1 / 0
-
-        @gen.coroutine
-        def f2():
-            try:
-                yield f1()
-            except ZeroDivisionError:
-                raise KeyError()
-
-        future = f2()
-        with self.assertRaises(KeyError):
-            yield future
-        self.finished = True
-
-    @gen_test
-    def test_swallow_yieldpoint_exception(self):
-        # Test exception handling: a coroutine can catch an exception
-        # raised by a yield point and not raise a different one.
-        @gen.coroutine
-        def f1():
-            1 / 0
-
-        @gen.coroutine
-        def f2():
-            try:
-                yield f1()
-            except ZeroDivisionError:
-                raise gen.Return(42)
-
-        result = yield f2()
-        self.assertEqual(result, 42)
-        self.finished = True
-
-    @gen_test
-    def test_replace_context_exception(self):
-        with ignore_deprecation():
-            # Test exception handling: exceptions thrown into the stack context
-            # can be caught and replaced.
-            # Note that this test and the following are for behavior that is
-            # not really supported any more:  coroutines no longer create a
-            # stack context automatically; but one is created after the first
-            # YieldPoint (i.e. not a Future).
-            @gen.coroutine
-            def f2():
-                (yield gen.Callback(1))()
-                yield gen.Wait(1)
-                self.io_loop.add_callback(lambda: 1 / 0)
-                try:
-                    yield gen.Task(self.io_loop.add_timeout,
-                                   self.io_loop.time() + 10)
-                except ZeroDivisionError:
-                    raise KeyError()
-
-            future = f2()
-            with self.assertRaises(KeyError):
-                yield future
-            self.finished = True
-
-    @gen_test
-    def test_swallow_context_exception(self):
-        with ignore_deprecation():
-            # Test exception handling: exceptions thrown into the stack context
-            # can be caught and ignored.
-            @gen.coroutine
-            def f2():
-                (yield gen.Callback(1))()
-                yield gen.Wait(1)
-                self.io_loop.add_callback(lambda: 1 / 0)
-                try:
-                    yield gen.Task(self.io_loop.add_timeout,
-                                   self.io_loop.time() + 10)
-                except ZeroDivisionError:
-                    raise gen.Return(42)
-
-            result = yield f2()
-            self.assertEqual(result, 42)
-            self.finished = True
-
-    @gen_test
-    def test_moment(self):
-        calls = []
-
-        @gen.coroutine
-        def f(name, yieldable):
-            for i in range(5):
-                calls.append(name)
-                yield yieldable
-        # First, confirm the behavior without moment: each coroutine
-        # monopolizes the event loop until it finishes.
-        immediate = Future()
-        immediate.set_result(None)
-        yield [f('a', immediate), f('b', immediate)]
-        self.assertEqual(''.join(calls), 'aaaaabbbbb')
-
-        # With moment, they take turns.
-        calls = []
-        yield [f('a', gen.moment), f('b', gen.moment)]
-        self.assertEqual(''.join(calls), 'ababababab')
-        self.finished = True
-
-        calls = []
-        yield [f('a', gen.moment), f('b', immediate)]
-        self.assertEqual(''.join(calls), 'abbbbbaaaa')
-
-    @gen_test
-    def test_sleep(self):
-        yield gen.sleep(0.01)
-        self.finished = True
-
-    @skipBefore33
-    @gen_test
-    def test_py3_leak_exception_context(self):
-        class LeakedException(Exception):
-            pass
-
-        @gen.coroutine
-        def inner(iteration):
-            raise LeakedException(iteration)
-
-        try:
-            yield inner(1)
-        except LeakedException as e:
-            self.assertEqual(str(e), "1")
-            self.assertIsNone(e.__context__)
-
-        try:
-            yield inner(2)
-        except LeakedException as e:
-            self.assertEqual(str(e), "2")
-            self.assertIsNone(e.__context__)
-
-        self.finished = True
-
-    @skipNotCPython
-    @unittest.skipIf((3,) < sys.version_info < (3, 6),
-                     "asyncio.Future has reference cycles")
-    def test_coroutine_refcounting(self):
-        # On CPython, tasks and their arguments should be released immediately
-        # without waiting for garbage collection.
-        @gen.coroutine
-        def inner():
-            class Foo(object):
-                pass
-            local_var = Foo()
-            self.local_ref = weakref.ref(local_var)
-            yield gen.coroutine(lambda: None)()
-            raise ValueError('Some error')
-
-        @gen.coroutine
-        def inner2():
-            try:
-                yield inner()
-            except ValueError:
-                pass
-
-        self.io_loop.run_sync(inner2, timeout=3)
-
-        self.assertIs(self.local_ref(), None)
-        self.finished = True
-
-    @unittest.skipIf(sys.version_info < (3,),
-                     "test only relevant with asyncio Futures")
-    def test_asyncio_future_debug_info(self):
-        self.finished = True
-        # Enable debug mode
-        asyncio_loop = asyncio.get_event_loop()
-        self.addCleanup(asyncio_loop.set_debug, asyncio_loop.get_debug())
-        asyncio_loop.set_debug(True)
-
-        def f():
-            yield gen.moment
-
-        coro = gen.coroutine(f)()
-        self.assertIsInstance(coro, asyncio.Future)
-        # We expect the coroutine repr() to show the place where
-        # it was instantiated
-        expected = ("created at %s:%d"
-                    % (__file__, f.__code__.co_firstlineno + 3))
-        actual = repr(coro)
-        self.assertIn(expected, actual)
-
-    @unittest.skipIf(asyncio is None, "asyncio module not present")
-    @gen_test
-    def test_asyncio_gather(self):
-        # This demonstrates that tornado coroutines can be understood
-        # by asyncio (This failed prior to Tornado 5.0).
-        @gen.coroutine
-        def f():
-            yield gen.moment
-            raise gen.Return(1)
-
-        ret = yield asyncio.gather(f(), f())
-        self.assertEqual(ret, [1, 1])
-        self.finished = True
-
-
-class GenSequenceHandler(RequestHandler):
-    with ignore_deprecation():
-        @asynchronous
-        @gen.engine
-        def get(self):
-            # The outer ignore_deprecation applies at definition time.
-            # We need another for serving time.
-            with ignore_deprecation():
-                self.io_loop = self.request.connection.stream.io_loop
-                self.io_loop.add_callback((yield gen.Callback("k1")))
-                yield gen.Wait("k1")
-                self.write("1")
-                self.io_loop.add_callback((yield gen.Callback("k2")))
-                yield gen.Wait("k2")
-                self.write("2")
-                # reuse an old key
-                self.io_loop.add_callback((yield gen.Callback("k1")))
-                yield gen.Wait("k1")
-                self.finish("3")
-
-
-class GenCoroutineSequenceHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        yield gen.moment
-        self.write("1")
-        yield gen.moment
-        self.write("2")
-        yield gen.moment
-        self.finish("3")
-
-
-class GenCoroutineUnfinishedSequenceHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        yield gen.moment
-        self.write("1")
-        yield gen.moment
-        self.write("2")
-        yield gen.moment
-        # just write, don't finish
-        self.write("3")
-
-
-class GenTaskHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        client = AsyncHTTPClient()
-        with ignore_deprecation():
-            response = yield gen.Task(client.fetch, self.get_argument('url'))
-        response.rethrow()
-        self.finish(b"got response: " + response.body)
-
-
-class GenExceptionHandler(RequestHandler):
-    with ignore_deprecation():
-        @asynchronous
-        @gen.engine
-        def get(self):
-            # This test depends on the order of the two decorators.
-            io_loop = self.request.connection.stream.io_loop
-            yield gen.Task(io_loop.add_callback)
-            raise Exception("oops")
-
-
-class GenCoroutineExceptionHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        # This test depends on the order of the two decorators.
-        io_loop = self.request.connection.stream.io_loop
-        yield gen.Task(io_loop.add_callback)
-        raise Exception("oops")
-
-
-class GenYieldExceptionHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        io_loop = self.request.connection.stream.io_loop
-        # Test the interaction of the two stack_contexts.
-        with ignore_deprecation():
-            def fail_task(callback):
-                io_loop.add_callback(lambda: 1 / 0)
-            try:
-                yield gen.Task(fail_task)
-                raise Exception("did not get expected exception")
-            except ZeroDivisionError:
-                self.finish('ok')
-
-
-# "Undecorated" here refers to the absence of @asynchronous.
-class UndecoratedCoroutinesHandler(RequestHandler):
-    @gen.coroutine
-    def prepare(self):
-        self.chunks = []
-        yield gen.moment
-        self.chunks.append('1')
-
-    @gen.coroutine
-    def get(self):
-        self.chunks.append('2')
-        yield gen.moment
-        self.chunks.append('3')
-        yield gen.moment
-        self.write(''.join(self.chunks))
-
-
-class AsyncPrepareErrorHandler(RequestHandler):
-    @gen.coroutine
-    def prepare(self):
-        yield gen.moment
-        raise HTTPError(403)
-
-    def get(self):
-        self.finish('ok')
-
-
-class NativeCoroutineHandler(RequestHandler):
-    if sys.version_info > (3, 5):
-        exec(textwrap.dedent("""
-        async def get(self):
-            import asyncio
-            await asyncio.sleep(0)
-            self.write("ok")
-        """))
-
-
-class GenWebTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([
-            ('/sequence', GenSequenceHandler),
-            ('/coroutine_sequence', GenCoroutineSequenceHandler),
-            ('/coroutine_unfinished_sequence',
-             GenCoroutineUnfinishedSequenceHandler),
-            ('/task', GenTaskHandler),
-            ('/exception', GenExceptionHandler),
-            ('/coroutine_exception', GenCoroutineExceptionHandler),
-            ('/yield_exception', GenYieldExceptionHandler),
-            ('/undecorated_coroutine', UndecoratedCoroutinesHandler),
-            ('/async_prepare_error', AsyncPrepareErrorHandler),
-            ('/native_coroutine', NativeCoroutineHandler),
-        ])
-
-    def test_sequence_handler(self):
-        response = self.fetch('/sequence')
-        self.assertEqual(response.body, b"123")
-
-    def test_coroutine_sequence_handler(self):
-        response = self.fetch('/coroutine_sequence')
-        self.assertEqual(response.body, b"123")
-
-    def test_coroutine_unfinished_sequence_handler(self):
-        response = self.fetch('/coroutine_unfinished_sequence')
-        self.assertEqual(response.body, b"123")
-
-    def test_task_handler(self):
-        response = self.fetch('/task?url=%s' % url_escape(self.get_url('/sequence')))
-        self.assertEqual(response.body, b"got response: 123")
-
-    def test_exception_handler(self):
-        # Make sure we get an error and not a timeout
-        with ExpectLog(app_log, "Uncaught exception GET /exception"):
-            response = self.fetch('/exception')
-        self.assertEqual(500, response.code)
-
-    def test_coroutine_exception_handler(self):
-        # Make sure we get an error and not a timeout
-        with ExpectLog(app_log, "Uncaught exception GET /coroutine_exception"):
-            response = self.fetch('/coroutine_exception')
-        self.assertEqual(500, response.code)
-
-    def test_yield_exception_handler(self):
-        response = self.fetch('/yield_exception')
-        self.assertEqual(response.body, b'ok')
-
-    def test_undecorated_coroutines(self):
-        response = self.fetch('/undecorated_coroutine')
-        self.assertEqual(response.body, b'123')
-
-    def test_async_prepare_error_handler(self):
-        response = self.fetch('/async_prepare_error')
-        self.assertEqual(response.code, 403)
-
-    @skipBefore35
-    def test_native_coroutine_handler(self):
-        response = self.fetch('/native_coroutine')
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.body, b'ok')
-
-
-class WithTimeoutTest(AsyncTestCase):
-    @gen_test
-    def test_timeout(self):
-        with self.assertRaises(gen.TimeoutError):
-            yield gen.with_timeout(datetime.timedelta(seconds=0.1),
-                                   Future())
-
-    @gen_test
-    def test_completes_before_timeout(self):
-        future = Future()
-        self.io_loop.add_timeout(datetime.timedelta(seconds=0.1),
-                                 lambda: future.set_result('asdf'))
-        result = yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                        future)
-        self.assertEqual(result, 'asdf')
-
-    @gen_test
-    def test_fails_before_timeout(self):
-        future = Future()
-        self.io_loop.add_timeout(
-            datetime.timedelta(seconds=0.1),
-            lambda: future.set_exception(ZeroDivisionError()))
-        with self.assertRaises(ZeroDivisionError):
-            yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                   future)
-
-    @gen_test
-    def test_already_resolved(self):
-        future = Future()
-        future.set_result('asdf')
-        result = yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                        future)
-        self.assertEqual(result, 'asdf')
-
-    @unittest.skipIf(futures is None, 'futures module not present')
-    @gen_test
-    def test_timeout_concurrent_future(self):
-        # A concurrent future that does not resolve before the timeout.
-        with futures.ThreadPoolExecutor(1) as executor:
-            with self.assertRaises(gen.TimeoutError):
-                yield gen.with_timeout(self.io_loop.time(),
-                                       executor.submit(time.sleep, 0.1))
-
-    @unittest.skipIf(futures is None, 'futures module not present')
-    @gen_test
-    def test_completed_concurrent_future(self):
-        # A concurrent future that is resolved before we even submit it
-        # to with_timeout.
-        with futures.ThreadPoolExecutor(1) as executor:
-            f = executor.submit(lambda: None)
-            f.result()  # wait for completion
-            yield gen.with_timeout(datetime.timedelta(seconds=3600), f)
-
-    @unittest.skipIf(futures is None, 'futures module not present')
-    @gen_test
-    def test_normal_concurrent_future(self):
-        # A conccurrent future that resolves while waiting for the timeout.
-        with futures.ThreadPoolExecutor(1) as executor:
-            yield gen.with_timeout(datetime.timedelta(seconds=3600),
-                                   executor.submit(lambda: time.sleep(0.01)))
-
-
-class WaitIteratorTest(AsyncTestCase):
-    @gen_test
-    def test_empty_iterator(self):
-        g = gen.WaitIterator()
-        self.assertTrue(g.done(), 'empty generator iterated')
-
-        with self.assertRaises(ValueError):
-            g = gen.WaitIterator(False, bar=False)
-
-        self.assertEqual(g.current_index, None, "bad nil current index")
-        self.assertEqual(g.current_future, None, "bad nil current future")
-
-    @gen_test
-    def test_already_done(self):
-        f1 = Future()
-        f2 = Future()
-        f3 = Future()
-        f1.set_result(24)
-        f2.set_result(42)
-        f3.set_result(84)
-
-        g = gen.WaitIterator(f1, f2, f3)
-        i = 0
-        while not g.done():
-            r = yield g.next()
-            # Order is not guaranteed, but the current implementation
-            # preserves ordering of already-done Futures.
-            if i == 0:
-                self.assertEqual(g.current_index, 0)
-                self.assertIs(g.current_future, f1)
-                self.assertEqual(r, 24)
-            elif i == 1:
-                self.assertEqual(g.current_index, 1)
-                self.assertIs(g.current_future, f2)
-                self.assertEqual(r, 42)
-            elif i == 2:
-                self.assertEqual(g.current_index, 2)
-                self.assertIs(g.current_future, f3)
-                self.assertEqual(r, 84)
-            i += 1
-
-        self.assertEqual(g.current_index, None, "bad nil current index")
-        self.assertEqual(g.current_future, None, "bad nil current future")
-
-        dg = gen.WaitIterator(f1=f1, f2=f2)
-
-        while not dg.done():
-            dr = yield dg.next()
-            if dg.current_index == "f1":
-                self.assertTrue(dg.current_future == f1 and dr == 24,
-                                "WaitIterator dict status incorrect")
-            elif dg.current_index == "f2":
-                self.assertTrue(dg.current_future == f2 and dr == 42,
-                                "WaitIterator dict status incorrect")
-            else:
-                self.fail("got bad WaitIterator index {}".format(
-                    dg.current_index))
-
-            i += 1
-
-        self.assertEqual(dg.current_index, None, "bad nil current index")
-        self.assertEqual(dg.current_future, None, "bad nil current future")
-
-    def finish_coroutines(self, iteration, futures):
-        if iteration == 3:
-            futures[2].set_result(24)
-        elif iteration == 5:
-            futures[0].set_exception(ZeroDivisionError())
-        elif iteration == 8:
-            futures[1].set_result(42)
-            futures[3].set_result(84)
-
-        if iteration < 8:
-            self.io_loop.add_callback(self.finish_coroutines, iteration + 1, futures)
-
-    @gen_test
-    def test_iterator(self):
-        futures = [Future(), Future(), Future(), Future()]
-
-        self.finish_coroutines(0, futures)
-
-        g = gen.WaitIterator(*futures)
-
-        i = 0
-        while not g.done():
-            try:
-                r = yield g.next()
-            except ZeroDivisionError:
-                self.assertIs(g.current_future, futures[0],
-                              'exception future invalid')
-            else:
-                if i == 0:
-                    self.assertEqual(r, 24, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 2, 'wrong index')
-                elif i == 2:
-                    self.assertEqual(r, 42, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 1, 'wrong index')
-                elif i == 3:
-                    self.assertEqual(r, 84, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 3, 'wrong index')
-            i += 1
-
-    @skipBefore35
-    @gen_test
-    def test_iterator_async_await(self):
-        # Recreate the previous test with py35 syntax. It's a little clunky
-        # because of the way the previous test handles an exception on
-        # a single iteration.
-        futures = [Future(), Future(), Future(), Future()]
-        self.finish_coroutines(0, futures)
-        self.finished = False
-
-        namespace = exec_test(globals(), locals(), """
-        async def f():
-            i = 0
-            g = gen.WaitIterator(*futures)
-            try:
-                async for r in g:
-                    if i == 0:
-                        self.assertEqual(r, 24, 'iterator value incorrect')
-                        self.assertEqual(g.current_index, 2, 'wrong index')
-                    else:
-                        raise Exception("expected exception on iteration 1")
-                    i += 1
-            except ZeroDivisionError:
-                i += 1
-            async for r in g:
-                if i == 2:
-                    self.assertEqual(r, 42, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 1, 'wrong index')
-                elif i == 3:
-                    self.assertEqual(r, 84, 'iterator value incorrect')
-                    self.assertEqual(g.current_index, 3, 'wrong index')
-                else:
-                    raise Exception("didn't expect iteration %d" % i)
-                i += 1
-            self.finished = True
-        """)
-        yield namespace['f']()
-        self.assertTrue(self.finished)
-
-    @gen_test
-    def test_no_ref(self):
-        # In this usage, there is no direct hard reference to the
-        # WaitIterator itself, only the Future it returns. Since
-        # WaitIterator uses weak references internally to improve GC
-        # performance, this used to cause problems.
-        yield gen.with_timeout(datetime.timedelta(seconds=0.1),
-                               gen.WaitIterator(gen.sleep(0)).next())
-
-
-class RunnerGCTest(AsyncTestCase):
-    def is_pypy3(self):
-        return (platform.python_implementation() == 'PyPy' and
-                sys.version_info > (3,))
-
-    @gen_test
-    def test_gc(self):
-        # Github issue 1769: Runner objects can get GCed unexpectedly
-        # while their future is alive.
-        weakref_scope = [None]
-
-        def callback():
-            gc.collect(2)
-            weakref_scope[0]().set_result(123)
-
-        @gen.coroutine
-        def tester():
-            fut = Future()
-            weakref_scope[0] = weakref.ref(fut)
-            self.io_loop.add_callback(callback)
-            yield fut
-
-        yield gen.with_timeout(
-            datetime.timedelta(seconds=0.2),
-            tester()
-        )
-
-    def test_gc_infinite_coro(self):
-        # Github issue 2229: suspended coroutines should be GCed when
-        # their loop is closed, even if they're involved in a reference
-        # cycle.
-        if IOLoop.configured_class().__name__.endswith('TwistedIOLoop'):
-            raise unittest.SkipTest("Test may fail on TwistedIOLoop")
-
-        loop = self.get_new_ioloop()
-        result = []
-        wfut = []
-
-        @gen.coroutine
-        def infinite_coro():
-            try:
-                while True:
-                    yield gen.sleep(1e-3)
-                    result.append(True)
-            finally:
-                # coroutine finalizer
-                result.append(None)
-
-        @gen.coroutine
-        def do_something():
-            fut = infinite_coro()
-            fut._refcycle = fut
-            wfut.append(weakref.ref(fut))
-            yield gen.sleep(0.2)
-
-        loop.run_sync(do_something)
-        loop.close()
-        gc.collect()
-        # Future was collected
-        self.assertIs(wfut[0](), None)
-        # At least one wakeup
-        self.assertGreaterEqual(len(result), 2)
-        if not self.is_pypy3():
-            # coroutine finalizer was called (not on PyPy3 apparently)
-            self.assertIs(result[-1], None)
-
-    @skipBefore35
-    def test_gc_infinite_async_await(self):
-        # Same as test_gc_infinite_coro, but with a `async def` function
-        import asyncio
-
-        namespace = exec_test(globals(), locals(), """
-        async def infinite_coro(result):
-            try:
-                while True:
-                    await gen.sleep(1e-3)
-                    result.append(True)
-            finally:
-                # coroutine finalizer
-                result.append(None)
-        """)
-
-        infinite_coro = namespace['infinite_coro']
-        loop = self.get_new_ioloop()
-        result = []
-        wfut = []
-
-        @gen.coroutine
-        def do_something():
-            fut = asyncio.get_event_loop().create_task(infinite_coro(result))
-            fut._refcycle = fut
-            wfut.append(weakref.ref(fut))
-            yield gen.sleep(0.2)
-
-        loop.run_sync(do_something)
-        with ExpectLog('asyncio', "Task was destroyed but it is pending"):
-            loop.close()
-            gc.collect()
-        # Future was collected
-        self.assertIs(wfut[0](), None)
-        # At least one wakeup and one finally
-        self.assertGreaterEqual(len(result), 2)
-        if not self.is_pypy3():
-            # coroutine finalizer was called (not on PyPy3 apparently)
-            self.assertIs(result[-1], None)
-
-    def test_multi_moment(self):
-        # Test gen.multi with moment
-        # now that it's not a real Future
-        @gen.coroutine
-        def wait_a_moment():
-            result = yield gen.multi([gen.moment, gen.moment])
-            raise gen.Return(result)
-
-        loop = self.get_new_ioloop()
-        result = loop.run_sync(wait_a_moment)
-        self.assertEqual(result, [None, None])
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/lib/tornado/test/gettext_translations/extract_me.py b/lib/tornado/test/gettext_translations/extract_me.py
deleted file mode 100755
index 283c13f4..00000000
--- a/lib/tornado/test/gettext_translations/extract_me.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# flake8: noqa
-# Dummy source file to allow creation of the initial .po file in the
-# same way as a real project.  I'm not entirely sure about the real
-# workflow here, but this seems to work.
-#
-# 1) xgettext --language=Python --keyword=_:1,2 --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3 extract_me.py -o tornado_test.po
-# 2) Edit tornado_test.po, setting CHARSET, Plural-Forms and setting msgstr
-# 3) msgfmt tornado_test.po -o tornado_test.mo
-# 4) Put the file in the proper location: $LANG/LC_MESSAGES
-
-from __future__ import absolute_import, division, print_function
-_("school")
-pgettext("law", "right")
-pgettext("good", "right")
-pgettext("organization", "club", "clubs", 1)
-pgettext("stick", "club", "clubs", 1)
diff --git a/lib/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.mo b/lib/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.mo
deleted file mode 100755
index a97bf9c5..00000000
Binary files a/lib/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.mo and /dev/null differ
diff --git a/lib/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po b/lib/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po
deleted file mode 100755
index 88d72c86..00000000
--- a/lib/tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po
+++ /dev/null
@@ -1,47 +0,0 @@
-# SOME DESCRIPTIVE TITLE.
-# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
-# FIRST AUTHOR , YEAR.
-#
-#, fuzzy
-msgid ""
-msgstr ""
-"Project-Id-Version: PACKAGE VERSION\n"
-"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2015-01-27 11:05+0300\n"
-"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
-"Last-Translator: FULL NAME \n"
-"Language-Team: LANGUAGE \n"
-"Language: \n"
-"MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=utf-8\n"
-"Content-Transfer-Encoding: 8bit\n"
-"Plural-Forms: nplurals=2; plural=(n > 1);\n"
-
-#: extract_me.py:11
-msgid "school"
-msgstr "école"
-
-#: extract_me.py:12
-msgctxt "law"
-msgid "right"
-msgstr "le droit"
-
-#: extract_me.py:13
-msgctxt "good"
-msgid "right"
-msgstr "le bien"
-
-#: extract_me.py:14
-msgctxt "organization"
-msgid "club"
-msgid_plural "clubs"
-msgstr[0] "le club"
-msgstr[1] "les clubs"
-
-#: extract_me.py:15
-msgctxt "stick"
-msgid "club"
-msgid_plural "clubs"
-msgstr[0] "le bâton"
-msgstr[1] "les bâtons"
diff --git a/lib/tornado/test/http1connection_test.py b/lib/tornado/test/http1connection_test.py
deleted file mode 100755
index 8aaaaf35..00000000
--- a/lib/tornado/test/http1connection_test.py
+++ /dev/null
@@ -1,61 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-import socket
-
-from tornado.http1connection import HTTP1Connection
-from tornado.httputil import HTTPMessageDelegate
-from tornado.iostream import IOStream
-from tornado.locks import Event
-from tornado.netutil import add_accept_handler
-from tornado.testing import AsyncTestCase, bind_unused_port, gen_test
-
-
-class HTTP1ConnectionTest(AsyncTestCase):
-    def setUp(self):
-        super(HTTP1ConnectionTest, self).setUp()
-        self.asyncSetUp()
-
-    @gen_test
-    def asyncSetUp(self):
-        listener, port = bind_unused_port()
-        event = Event()
-
-        def accept_callback(conn, addr):
-            self.server_stream = IOStream(conn)
-            self.addCleanup(self.server_stream.close)
-            event.set()
-
-        add_accept_handler(listener, accept_callback)
-        self.client_stream = IOStream(socket.socket())
-        self.addCleanup(self.client_stream.close)
-        yield [self.client_stream.connect(('127.0.0.1', port)),
-               event.wait()]
-        self.io_loop.remove_handler(listener)
-        listener.close()
-
-    @gen_test
-    def test_http10_no_content_length(self):
-        # Regression test for a bug in which can_keep_alive would crash
-        # for an HTTP/1.0 (not 1.1) response with no content-length.
-        conn = HTTP1Connection(self.client_stream, True)
-        self.server_stream.write(b"HTTP/1.0 200 Not Modified\r\n\r\nhello")
-        self.server_stream.close()
-
-        event = Event()
-        test = self
-        body = []
-
-        class Delegate(HTTPMessageDelegate):
-            def headers_received(self, start_line, headers):
-                test.code = start_line.code
-
-            def data_received(self, data):
-                body.append(data)
-
-            def finish(self):
-                event.set()
-
-        yield conn.read_response(Delegate())
-        yield event.wait()
-        self.assertEqual(self.code, 200)
-        self.assertEqual(b''.join(body), b'hello')
diff --git a/lib/tornado/test/httpclient_test.py b/lib/tornado/test/httpclient_test.py
deleted file mode 100755
index 35426a7d..00000000
--- a/lib/tornado/test/httpclient_test.py
+++ /dev/null
@@ -1,718 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, division, print_function
-
-import base64
-import binascii
-from contextlib import closing
-import copy
-import sys
-import threading
-import datetime
-from io import BytesIO
-import time
-import unicodedata
-
-from tornado.escape import utf8, native_str
-from tornado import gen
-from tornado.httpclient import HTTPRequest, HTTPResponse, _RequestProxy, HTTPError, HTTPClient
-from tornado.httpserver import HTTPServer
-from tornado.ioloop import IOLoop
-from tornado.iostream import IOStream
-from tornado.log import gen_log
-from tornado import netutil
-from tornado.stack_context import ExceptionStackContext, NullContext
-from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
-from tornado.test.util import unittest, skipOnTravis, ignore_deprecation
-from tornado.web import Application, RequestHandler, url
-from tornado.httputil import format_timestamp, HTTPHeaders
-
-
-class HelloWorldHandler(RequestHandler):
-    def get(self):
-        name = self.get_argument("name", "world")
-        self.set_header("Content-Type", "text/plain")
-        self.finish("Hello %s!" % name)
-
-
-class PostHandler(RequestHandler):
-    def post(self):
-        self.finish("Post arg1: %s, arg2: %s" % (
-            self.get_argument("arg1"), self.get_argument("arg2")))
-
-
-class PutHandler(RequestHandler):
-    def put(self):
-        self.write("Put body: ")
-        self.write(self.request.body)
-
-
-class RedirectHandler(RequestHandler):
-    def prepare(self):
-        self.write('redirects can have bodies too')
-        self.redirect(self.get_argument("url"),
-                      status=int(self.get_argument("status", "302")))
-
-
-class ChunkHandler(RequestHandler):
-    @gen.coroutine
-    def get(self):
-        self.write("asdf")
-        self.flush()
-        # Wait a bit to ensure the chunks are sent and received separately.
-        yield gen.sleep(0.01)
-        self.write("qwer")
-
-
-class AuthHandler(RequestHandler):
-    def get(self):
-        self.finish(self.request.headers["Authorization"])
-
-
-class CountdownHandler(RequestHandler):
-    def get(self, count):
-        count = int(count)
-        if count > 0:
-            self.redirect(self.reverse_url("countdown", count - 1))
-        else:
-            self.write("Zero")
-
-
-class EchoPostHandler(RequestHandler):
-    def post(self):
-        self.write(self.request.body)
-
-
-class UserAgentHandler(RequestHandler):
-    def get(self):
-        self.write(self.request.headers.get('User-Agent', 'User agent not set'))
-
-
-class ContentLength304Handler(RequestHandler):
-    def get(self):
-        self.set_status(304)
-        self.set_header('Content-Length', 42)
-
-    def _clear_headers_for_304(self):
-        # Tornado strips content-length from 304 responses, but here we
-        # want to simulate servers that include the headers anyway.
-        pass
-
-
-class PatchHandler(RequestHandler):
-
-    def patch(self):
-        "Return the request payload - so we can check it is being kept"
-        self.write(self.request.body)
-
-
-class AllMethodsHandler(RequestHandler):
-    SUPPORTED_METHODS = RequestHandler.SUPPORTED_METHODS + ('OTHER',)
-
-    def method(self):
-        self.write(self.request.method)
-
-    get = post = put = delete = options = patch = other = method
-
-
-class SetHeaderHandler(RequestHandler):
-    def get(self):
-        # Use get_arguments for keys to get strings, but
-        # request.arguments for values to get bytes.
-        for k, v in zip(self.get_arguments('k'),
-                        self.request.arguments['v']):
-            self.set_header(k, v)
-
-# These tests end up getting run redundantly: once here with the default
-# HTTPClient implementation, and then again in each implementation's own
-# test suite.
-
-
-class HTTPClientCommonTestCase(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([
-            url("/hello", HelloWorldHandler),
-            url("/post", PostHandler),
-            url("/put", PutHandler),
-            url("/redirect", RedirectHandler),
-            url("/chunk", ChunkHandler),
-            url("/auth", AuthHandler),
-            url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
-            url("/echopost", EchoPostHandler),
-            url("/user_agent", UserAgentHandler),
-            url("/304_with_content_length", ContentLength304Handler),
-            url("/all_methods", AllMethodsHandler),
-            url('/patch', PatchHandler),
-            url('/set_header', SetHeaderHandler),
-        ], gzip=True)
-
-    def test_patch_receives_payload(self):
-        body = b"some patch data"
-        response = self.fetch("/patch", method='PATCH', body=body)
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.body, body)
-
-    @skipOnTravis
-    def test_hello_world(self):
-        response = self.fetch("/hello")
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.headers["Content-Type"], "text/plain")
-        self.assertEqual(response.body, b"Hello world!")
-        self.assertEqual(int(response.request_time), 0)
-
-        response = self.fetch("/hello?name=Ben")
-        self.assertEqual(response.body, b"Hello Ben!")
-
-    def test_streaming_callback(self):
-        # streaming_callback is also tested in test_chunked
-        chunks = []
-        response = self.fetch("/hello",
-                              streaming_callback=chunks.append)
-        # with streaming_callback, data goes to the callback and not response.body
-        self.assertEqual(chunks, [b"Hello world!"])
-        self.assertFalse(response.body)
-
-    def test_post(self):
-        response = self.fetch("/post", method="POST",
-                              body="arg1=foo&arg2=bar")
-        self.assertEqual(response.code, 200)
-        self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
-
-    def test_chunked(self):
-        response = self.fetch("/chunk")
-        self.assertEqual(response.body, b"asdfqwer")
-
-        chunks = []
-        response = self.fetch("/chunk",
-                              streaming_callback=chunks.append)
-        self.assertEqual(chunks, [b"asdf", b"qwer"])
-        self.assertFalse(response.body)
-
-    def test_chunked_close(self):
-        # test case in which chunks spread read-callback processing
-        # over several ioloop iterations, but the connection is already closed.
-        sock, port = bind_unused_port()
-        with closing(sock):
-            @gen.coroutine
-            def accept_callback(conn, address):
-                # fake an HTTP server using chunked encoding where the final chunks
-                # and connection close all happen at once
-                stream = IOStream(conn)
-                request_data = yield stream.read_until(b"\r\n\r\n")
-                if b"HTTP/1." not in request_data:
-                    self.skipTest("requires HTTP/1.x")
-                yield stream.write(b"""\
-HTTP/1.1 200 OK
-Transfer-Encoding: chunked
-
-1
-1
-1
-2
-0
-
-""".replace(b"\n", b"\r\n"))
-                stream.close()
-            netutil.add_accept_handler(sock, accept_callback)
-            resp = self.fetch("http://127.0.0.1:%d/" % port)
-            resp.rethrow()
-            self.assertEqual(resp.body, b"12")
-            self.io_loop.remove_handler(sock.fileno())
-
-    def test_streaming_stack_context(self):
-        chunks = []
-        exc_info = []
-
-        def error_handler(typ, value, tb):
-            exc_info.append((typ, value, tb))
-            return True
-
-        def streaming_cb(chunk):
-            chunks.append(chunk)
-            if chunk == b'qwer':
-                1 / 0
-
-        with ignore_deprecation():
-            with ExceptionStackContext(error_handler):
-                self.fetch('/chunk', streaming_callback=streaming_cb)
-
-        self.assertEqual(chunks, [b'asdf', b'qwer'])
-        self.assertEqual(1, len(exc_info))
-        self.assertIs(exc_info[0][0], ZeroDivisionError)
-
-    def test_basic_auth(self):
-        # This test data appears in section 2 of RFC 7617.
-        self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
-                                    auth_password="open sesame").body,
-                         b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
-
-    def test_basic_auth_explicit_mode(self):
-        self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
-                                    auth_password="open sesame",
-                                    auth_mode="basic").body,
-                         b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
-
-    def test_basic_auth_unicode(self):
-        # This test data appears in section 2.1 of RFC 7617.
-        self.assertEqual(self.fetch("/auth", auth_username="test",
-                                    auth_password="123£").body,
-                         b"Basic dGVzdDoxMjPCow==")
-
-        # The standard mandates NFC. Give it a decomposed username
-        # and ensure it is normalized to composed form.
-        username = unicodedata.normalize("NFD", u"josé")
-        self.assertEqual(self.fetch("/auth",
-                                    auth_username=username,
-                                    auth_password="səcrət").body,
-                         b"Basic am9zw6k6c8mZY3LJmXQ=")
-
-    def test_unsupported_auth_mode(self):
-        # curl and simple clients handle errors a bit differently; the
-        # important thing is that they don't fall back to basic auth
-        # on an unknown mode.
-        with ExpectLog(gen_log, "uncaught exception", required=False):
-            with self.assertRaises((ValueError, HTTPError)):
-                self.fetch("/auth", auth_username="Aladdin",
-                           auth_password="open sesame",
-                           auth_mode="asdf",
-                           raise_error=True)
-
-    def test_follow_redirect(self):
-        response = self.fetch("/countdown/2", follow_redirects=False)
-        self.assertEqual(302, response.code)
-        self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
-
-        response = self.fetch("/countdown/2")
-        self.assertEqual(200, response.code)
-        self.assertTrue(response.effective_url.endswith("/countdown/0"))
-        self.assertEqual(b"Zero", response.body)
-
-    def test_credentials_in_url(self):
-        url = self.get_url("/auth").replace("http://", "http://me:secret@")
-        response = self.fetch(url)
-        self.assertEqual(b"Basic " + base64.b64encode(b"me:secret"),
-                         response.body)
-
-    def test_body_encoding(self):
-        unicode_body = u"\xe9"
-        byte_body = binascii.a2b_hex(b"e9")
-
-        # unicode string in body gets converted to utf8
-        response = self.fetch("/echopost", method="POST", body=unicode_body,
-                              headers={"Content-Type": "application/blah"})
-        self.assertEqual(response.headers["Content-Length"], "2")
-        self.assertEqual(response.body, utf8(unicode_body))
-
-        # byte strings pass through directly
-        response = self.fetch("/echopost", method="POST",
-                              body=byte_body,
-                              headers={"Content-Type": "application/blah"})
-        self.assertEqual(response.headers["Content-Length"], "1")
-        self.assertEqual(response.body, byte_body)
-
-        # Mixing unicode in headers and byte string bodies shouldn't
-        # break anything
-        response = self.fetch("/echopost", method="POST", body=byte_body,
-                              headers={"Content-Type": "application/blah"},
-                              user_agent=u"foo")
-        self.assertEqual(response.headers["Content-Length"], "1")
-        self.assertEqual(response.body, byte_body)
-
-    def test_types(self):
-        response = self.fetch("/hello")
-        self.assertEqual(type(response.body), bytes)
-        self.assertEqual(type(response.headers["Content-Type"]), str)
-        self.assertEqual(type(response.code), int)
-        self.assertEqual(type(response.effective_url), str)
-
-    def test_header_callback(self):
-        first_line = []
-        headers = {}
-        chunks = []
-
-        def header_callback(header_line):
-            if header_line.startswith('HTTP/1.1 101'):
-                # Upgrading to HTTP/2
-                pass
-            elif header_line.startswith('HTTP/'):
-                first_line.append(header_line)
-            elif header_line != '\r\n':
-                k, v = header_line.split(':', 1)
-                headers[k.lower()] = v.strip()
-
-        def streaming_callback(chunk):
-            # All header callbacks are run before any streaming callbacks,
-            # so the header data is available to process the data as it
-            # comes in.
-            self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
-            chunks.append(chunk)
-
-        self.fetch('/chunk', header_callback=header_callback,
-                   streaming_callback=streaming_callback)
-        self.assertEqual(len(first_line), 1, first_line)
-        self.assertRegexpMatches(first_line[0], 'HTTP/[0-9]\\.[0-9] 200.*\r\n')
-        self.assertEqual(chunks, [b'asdf', b'qwer'])
-
-    def test_header_callback_stack_context(self):
-        exc_info = []
-
-        def error_handler(typ, value, tb):
-            exc_info.append((typ, value, tb))
-            return True
-
-        def header_callback(header_line):
-            if header_line.lower().startswith('content-type:'):
-                1 / 0
-
-        with ignore_deprecation():
-            with ExceptionStackContext(error_handler):
-                self.fetch('/chunk', header_callback=header_callback)
-        self.assertEqual(len(exc_info), 1)
-        self.assertIs(exc_info[0][0], ZeroDivisionError)
-
-    @gen_test
-    def test_configure_defaults(self):
-        defaults = dict(user_agent='TestDefaultUserAgent', allow_ipv6=False)
-        # Construct a new instance of the configured client class
-        client = self.http_client.__class__(force_instance=True,
-                                            defaults=defaults)
-        try:
-            response = yield client.fetch(self.get_url('/user_agent'))
-            self.assertEqual(response.body, b'TestDefaultUserAgent')
-        finally:
-            client.close()
-
-    def test_header_types(self):
-        # Header values may be passed as character or utf8 byte strings,
-        # in a plain dictionary or an HTTPHeaders object.
-        # Keys must always be the native str type.
-        # All combinations should have the same results on the wire.
-        for value in [u"MyUserAgent", b"MyUserAgent"]:
-            for container in [dict, HTTPHeaders]:
-                headers = container()
-                headers['User-Agent'] = value
-                resp = self.fetch('/user_agent', headers=headers)
-                self.assertEqual(
-                    resp.body, b"MyUserAgent",
-                    "response=%r, value=%r, container=%r" %
-                    (resp.body, value, container))
-
-    def test_multi_line_headers(self):
-        # Multi-line http headers are rare but rfc-allowed
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
-        sock, port = bind_unused_port()
-        with closing(sock):
-            @gen.coroutine
-            def accept_callback(conn, address):
-                stream = IOStream(conn)
-                request_data = yield stream.read_until(b"\r\n\r\n")
-                if b"HTTP/1." not in request_data:
-                    self.skipTest("requires HTTP/1.x")
-                yield stream.write(b"""\
-HTTP/1.1 200 OK
-X-XSS-Protection: 1;
-\tmode=block
-
-""".replace(b"\n", b"\r\n"))
-                stream.close()
-
-            netutil.add_accept_handler(sock, accept_callback)
-            resp = self.fetch("http://127.0.0.1:%d/" % port)
-            resp.rethrow()
-            self.assertEqual(resp.headers['X-XSS-Protection'], "1; mode=block")
-            self.io_loop.remove_handler(sock.fileno())
-
-    def test_304_with_content_length(self):
-        # According to the spec 304 responses SHOULD NOT include
-        # Content-Length or other entity headers, but some servers do it
-        # anyway.
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
-        response = self.fetch('/304_with_content_length')
-        self.assertEqual(response.code, 304)
-        self.assertEqual(response.headers['Content-Length'], '42')
-
-    def test_final_callback_stack_context(self):
-        # The final callback should be run outside of the httpclient's
-        # stack_context.  We want to ensure that there is not stack_context
-        # between the user's callback and the IOLoop, so monkey-patch
-        # IOLoop.handle_callback_exception and disable the test harness's
-        # context with a NullContext.
-        # Note that this does not apply to secondary callbacks (header
-        # and streaming_callback), as errors there must be seen as errors
-        # by the http client so it can clean up the connection.
-        exc_info = []
-
-        def handle_callback_exception(callback):
-            exc_info.append(sys.exc_info())
-            self.stop()
-        self.io_loop.handle_callback_exception = handle_callback_exception
-        with NullContext():
-            with ignore_deprecation():
-                self.http_client.fetch(self.get_url('/hello'),
-                                       lambda response: 1 / 0)
-        self.wait()
-        self.assertEqual(exc_info[0][0], ZeroDivisionError)
-
-    @gen_test
-    def test_future_interface(self):
-        response = yield self.http_client.fetch(self.get_url('/hello'))
-        self.assertEqual(response.body, b'Hello world!')
-
-    @gen_test
-    def test_future_http_error(self):
-        with self.assertRaises(HTTPError) as context:
-            yield self.http_client.fetch(self.get_url('/notfound'))
-        self.assertEqual(context.exception.code, 404)
-        self.assertEqual(context.exception.response.code, 404)
-
-    @gen_test
-    def test_future_http_error_no_raise(self):
-        response = yield self.http_client.fetch(self.get_url('/notfound'), raise_error=False)
-        self.assertEqual(response.code, 404)
-
-    @gen_test
-    def test_reuse_request_from_response(self):
-        # The response.request attribute should be an HTTPRequest, not
-        # a _RequestProxy.
-        # This test uses self.http_client.fetch because self.fetch calls
-        # self.get_url on the input unconditionally.
-        url = self.get_url('/hello')
-        response = yield self.http_client.fetch(url)
-        self.assertEqual(response.request.url, url)
-        self.assertTrue(isinstance(response.request, HTTPRequest))
-        response2 = yield self.http_client.fetch(response.request)
-        self.assertEqual(response2.body, b'Hello world!')
-
-    def test_all_methods(self):
-        for method in ['GET', 'DELETE', 'OPTIONS']:
-            response = self.fetch('/all_methods', method=method)
-            self.assertEqual(response.body, utf8(method))
-        for method in ['POST', 'PUT', 'PATCH']:
-            response = self.fetch('/all_methods', method=method, body=b'')
-            self.assertEqual(response.body, utf8(method))
-        response = self.fetch('/all_methods', method='HEAD')
-        self.assertEqual(response.body, b'')
-        response = self.fetch('/all_methods', method='OTHER',
-                              allow_nonstandard_methods=True)
-        self.assertEqual(response.body, b'OTHER')
-
-    def test_body_sanity_checks(self):
-        # These methods require a body.
-        for method in ('POST', 'PUT', 'PATCH'):
-            with self.assertRaises(ValueError) as context:
-                self.fetch('/all_methods', method=method, raise_error=True)
-            self.assertIn('must not be None', str(context.exception))
-
-            resp = self.fetch('/all_methods', method=method,
-                              allow_nonstandard_methods=True)
-            self.assertEqual(resp.code, 200)
-
-        # These methods don't allow a body.
-        for method in ('GET', 'DELETE', 'OPTIONS'):
-            with self.assertRaises(ValueError) as context:
-                self.fetch('/all_methods', method=method, body=b'asdf', raise_error=True)
-            self.assertIn('must be None', str(context.exception))
-
-            # In most cases this can be overridden, but curl_httpclient
-            # does not allow body with a GET at all.
-            if method != 'GET':
-                self.fetch('/all_methods', method=method, body=b'asdf',
-                           allow_nonstandard_methods=True, raise_error=True)
-                self.assertEqual(resp.code, 200)
-
-    # This test causes odd failures with the combination of
-    # curl_httpclient (at least with the version of libcurl available
-    # on ubuntu 12.04), TwistedIOLoop, and epoll.  For POST (but not PUT),
-    # curl decides the response came back too soon and closes the connection
-    # to start again.  It does this *before* telling the socket callback to
-    # unregister the FD.  Some IOLoop implementations have special kernel
-    # integration to discover this immediately.  Tornado's IOLoops
-    # ignore errors on remove_handler to accommodate this behavior, but
-    # Twisted's reactor does not.  The removeReader call fails and so
-    # do all future removeAll calls (which our tests do at cleanup).
-    #
-    # def test_post_307(self):
-    #    response = self.fetch("/redirect?status=307&url=/post",
-    #                          method="POST", body=b"arg1=foo&arg2=bar")
-    #    self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
-
-    def test_put_307(self):
-        response = self.fetch("/redirect?status=307&url=/put",
-                              method="PUT", body=b"hello")
-        response.rethrow()
-        self.assertEqual(response.body, b"Put body: hello")
-
-    def test_non_ascii_header(self):
-        # Non-ascii headers are sent as latin1.
-        response = self.fetch("/set_header?k=foo&v=%E9")
-        response.rethrow()
-        self.assertEqual(response.headers["Foo"], native_str(u"\u00e9"))
-
-    def test_response_times(self):
-        # A few simple sanity checks of the response time fields to
-        # make sure they're using the right basis (between the
-        # wall-time and monotonic clocks).
-        start_time = time.time()
-        response = self.fetch("/hello")
-        response.rethrow()
-        self.assertGreaterEqual(response.request_time, 0)
-        self.assertLess(response.request_time, 1.0)
-        # A very crude check to make sure that start_time is based on
-        # wall time and not the monotonic clock.
-        self.assertLess(abs(response.start_time - start_time), 1.0)
-
-        for k, v in response.time_info.items():
-            self.assertTrue(0 <= v < 1.0, "time_info[%s] out of bounds: %s" % (k, v))
-
-
-class RequestProxyTest(unittest.TestCase):
-    def test_request_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/',
-                                          user_agent='foo'),
-                              dict())
-        self.assertEqual(proxy.user_agent, 'foo')
-
-    def test_default_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'),
-                              dict(network_interface='foo'))
-        self.assertEqual(proxy.network_interface, 'foo')
-
-    def test_both_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/',
-                                          proxy_host='foo'),
-                              dict(proxy_host='bar'))
-        self.assertEqual(proxy.proxy_host, 'foo')
-
-    def test_neither_set(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'),
-                              dict())
-        self.assertIs(proxy.auth_username, None)
-
-    def test_bad_attribute(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'),
-                              dict())
-        with self.assertRaises(AttributeError):
-            proxy.foo
-
-    def test_defaults_none(self):
-        proxy = _RequestProxy(HTTPRequest('http://example.com/'), None)
-        self.assertIs(proxy.auth_username, None)
-
-
-class HTTPResponseTestCase(unittest.TestCase):
-    def test_str(self):
-        response = HTTPResponse(HTTPRequest('http://example.com'),
-                                200, headers={}, buffer=BytesIO())
-        s = str(response)
-        self.assertTrue(s.startswith('HTTPResponse('))
-        self.assertIn('code=200', s)
-
-
-class SyncHTTPClientTest(unittest.TestCase):
-    def setUp(self):
-        if IOLoop.configured_class().__name__ == 'TwistedIOLoop':
-            # TwistedIOLoop only supports the global reactor, so we can't have
-            # separate IOLoops for client and server threads.
-            raise unittest.SkipTest(
-                'Sync HTTPClient not compatible with TwistedIOLoop')
-        self.server_ioloop = IOLoop()
-
-        @gen.coroutine
-        def init_server():
-            sock, self.port = bind_unused_port()
-            app = Application([('/', HelloWorldHandler)])
-            self.server = HTTPServer(app)
-            self.server.add_socket(sock)
-        self.server_ioloop.run_sync(init_server)
-
-        self.server_thread = threading.Thread(target=self.server_ioloop.start)
-        self.server_thread.start()
-
-        self.http_client = HTTPClient()
-
-    def tearDown(self):
-        def stop_server():
-            self.server.stop()
-            # Delay the shutdown of the IOLoop by several iterations because
-            # the server may still have some cleanup work left when
-            # the client finishes with the response (this is noticeable
-            # with http/2, which leaves a Future with an unexamined
-            # StreamClosedError on the loop).
-
-            @gen.coroutine
-            def slow_stop():
-                # The number of iterations is difficult to predict. Typically,
-                # one is sufficient, although sometimes it needs more.
-                for i in range(5):
-                    yield
-                self.server_ioloop.stop()
-            self.server_ioloop.add_callback(slow_stop)
-        self.server_ioloop.add_callback(stop_server)
-        self.server_thread.join()
-        self.http_client.close()
-        self.server_ioloop.close(all_fds=True)
-
-    def get_url(self, path):
-        return 'http://127.0.0.1:%d%s' % (self.port, path)
-
-    def test_sync_client(self):
-        response = self.http_client.fetch(self.get_url('/'))
-        self.assertEqual(b'Hello world!', response.body)
-
-    def test_sync_client_error(self):
-        # Synchronous HTTPClient raises errors directly; no need for
-        # response.rethrow()
-        with self.assertRaises(HTTPError) as assertion:
-            self.http_client.fetch(self.get_url('/notfound'))
-        self.assertEqual(assertion.exception.code, 404)
-
-
-class HTTPRequestTestCase(unittest.TestCase):
-    def test_headers(self):
-        request = HTTPRequest('http://example.com', headers={'foo': 'bar'})
-        self.assertEqual(request.headers, {'foo': 'bar'})
-
-    def test_headers_setter(self):
-        request = HTTPRequest('http://example.com')
-        request.headers = {'bar': 'baz'}
-        self.assertEqual(request.headers, {'bar': 'baz'})
-
-    def test_null_headers_setter(self):
-        request = HTTPRequest('http://example.com')
-        request.headers = None
-        self.assertEqual(request.headers, {})
-
-    def test_body(self):
-        request = HTTPRequest('http://example.com', body='foo')
-        self.assertEqual(request.body, utf8('foo'))
-
-    def test_body_setter(self):
-        request = HTTPRequest('http://example.com')
-        request.body = 'foo'
-        self.assertEqual(request.body, utf8('foo'))
-
-    def test_if_modified_since(self):
-        http_date = datetime.datetime.utcnow()
-        request = HTTPRequest('http://example.com', if_modified_since=http_date)
-        self.assertEqual(request.headers,
-                         {'If-Modified-Since': format_timestamp(http_date)})
-
-
-class HTTPErrorTestCase(unittest.TestCase):
-    def test_copy(self):
-        e = HTTPError(403)
-        e2 = copy.copy(e)
-        self.assertIsNot(e, e2)
-        self.assertEqual(e.code, e2.code)
-
-    def test_plain_error(self):
-        e = HTTPError(403)
-        self.assertEqual(str(e), "HTTP 403: Forbidden")
-        self.assertEqual(repr(e), "HTTP 403: Forbidden")
-
-    def test_error_with_response(self):
-        resp = HTTPResponse(HTTPRequest('http://example.com/'), 403)
-        with self.assertRaises(HTTPError) as cm:
-            resp.rethrow()
-        e = cm.exception
-        self.assertEqual(str(e), "HTTP 403: Forbidden")
-        self.assertEqual(repr(e), "HTTP 403: Forbidden")
diff --git a/lib/tornado/test/httpserver_test.py b/lib/tornado/test/httpserver_test.py
deleted file mode 100755
index acaf2a00..00000000
--- a/lib/tornado/test/httpserver_test.py
+++ /dev/null
@@ -1,1167 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-from tornado import gen, netutil
-from tornado.concurrent import Future
-from tornado.escape import json_decode, json_encode, utf8, _unicode, recursive_unicode, native_str
-from tornado.http1connection import HTTP1Connection
-from tornado.httpclient import HTTPError
-from tornado.httpserver import HTTPServer
-from tornado.httputil import HTTPHeaders, HTTPMessageDelegate, HTTPServerConnectionDelegate, ResponseStartLine  # noqa: E501
-from tornado.iostream import IOStream
-from tornado.locks import Event
-from tornado.log import gen_log
-from tornado.netutil import ssl_options_to_context
-from tornado.simple_httpclient import SimpleAsyncHTTPClient
-from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, ExpectLog, gen_test  # noqa: E501
-from tornado.test.util import unittest, skipOnTravis
-from tornado.web import Application, RequestHandler, stream_request_body
-
-from contextlib import closing
-import datetime
-import gzip
-import os
-import shutil
-import socket
-import ssl
-import sys
-import tempfile
-from io import BytesIO
-
-
-def read_stream_body(stream, callback):
-    """Reads an HTTP response from `stream` and runs callback with its
-    start_line, headers and body."""
-    chunks = []
-
-    class Delegate(HTTPMessageDelegate):
-        def headers_received(self, start_line, headers):
-            self.headers = headers
-            self.start_line = start_line
-
-        def data_received(self, chunk):
-            chunks.append(chunk)
-
-        def finish(self):
-            conn.detach()
-            callback((self.start_line, self.headers, b''.join(chunks)))
-    conn = HTTP1Connection(stream, True)
-    conn.read_response(Delegate())
-
-
-class HandlerBaseTestCase(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([('/', self.__class__.Handler)])
-
-    def fetch_json(self, *args, **kwargs):
-        response = self.fetch(*args, **kwargs)
-        response.rethrow()
-        return json_decode(response.body)
-
-
-class HelloWorldRequestHandler(RequestHandler):
-    def initialize(self, protocol="http"):
-        self.expected_protocol = protocol
-
-    def get(self):
-        if self.request.protocol != self.expected_protocol:
-            raise Exception("unexpected protocol")
-        self.finish("Hello world")
-
-    def post(self):
-        self.finish("Got %d bytes in POST" % len(self.request.body))
-
-
-# In pre-1.0 versions of openssl, SSLv23 clients always send SSLv2
-# ClientHello messages, which are rejected by SSLv3 and TLSv1
-# servers.  Note that while the OPENSSL_VERSION_INFO was formally
-# introduced in python3.2, it was present but undocumented in
-# python 2.7
-skipIfOldSSL = unittest.skipIf(
-    getattr(ssl, 'OPENSSL_VERSION_INFO', (0, 0)) < (1, 0),
-    "old version of ssl module and/or openssl")
-
-
-class BaseSSLTest(AsyncHTTPSTestCase):
-    def get_app(self):
-        return Application([('/', HelloWorldRequestHandler,
-                             dict(protocol="https"))])
-
-
-class SSLTestMixin(object):
-    def get_ssl_options(self):
-        return dict(ssl_version=self.get_ssl_version(),  # type: ignore
-                    **AsyncHTTPSTestCase.get_ssl_options())
-
-    def get_ssl_version(self):
-        raise NotImplementedError()
-
-    def test_ssl(self):
-        response = self.fetch('/')
-        self.assertEqual(response.body, b"Hello world")
-
-    def test_large_post(self):
-        response = self.fetch('/',
-                              method='POST',
-                              body='A' * 5000)
-        self.assertEqual(response.body, b"Got 5000 bytes in POST")
-
-    def test_non_ssl_request(self):
-        # Make sure the server closes the connection when it gets a non-ssl
-        # connection, rather than waiting for a timeout or otherwise
-        # misbehaving.
-        with ExpectLog(gen_log, '(SSL Error|uncaught exception)'):
-            with ExpectLog(gen_log, 'Uncaught exception', required=False):
-                with self.assertRaises((IOError, HTTPError)):
-                    self.fetch(
-                        self.get_url("/").replace('https:', 'http:'),
-                        request_timeout=3600,
-                        connect_timeout=3600,
-                        raise_error=True)
-
-    def test_error_logging(self):
-        # No stack traces are logged for SSL errors.
-        with ExpectLog(gen_log, 'SSL Error') as expect_log:
-            with self.assertRaises((IOError, HTTPError)):
-                self.fetch(self.get_url("/").replace("https:", "http:"),
-                           raise_error=True)
-        self.assertFalse(expect_log.logged_stack)
-
-# Python's SSL implementation differs significantly between versions.
-# For example, SSLv3 and TLSv1 throw an exception if you try to read
-# from the socket before the handshake is complete, but the default
-# of SSLv23 allows it.
-
-
-class SSLv23Test(BaseSSLTest, SSLTestMixin):
-    def get_ssl_version(self):
-        return ssl.PROTOCOL_SSLv23
-
-
-@skipIfOldSSL
-class SSLv3Test(BaseSSLTest, SSLTestMixin):
-    def get_ssl_version(self):
-        return ssl.PROTOCOL_SSLv3
-
-
-@skipIfOldSSL
-class TLSv1Test(BaseSSLTest, SSLTestMixin):
-    def get_ssl_version(self):
-        return ssl.PROTOCOL_TLSv1
-
-
-class SSLContextTest(BaseSSLTest, SSLTestMixin):
-    def get_ssl_options(self):
-        context = ssl_options_to_context(
-            AsyncHTTPSTestCase.get_ssl_options(self))
-        assert isinstance(context, ssl.SSLContext)
-        return context
-
-
-class BadSSLOptionsTest(unittest.TestCase):
-    def test_missing_arguments(self):
-        application = Application()
-        self.assertRaises(KeyError, HTTPServer, application, ssl_options={
-            "keyfile": "/__missing__.crt",
-        })
-
-    def test_missing_key(self):
-        """A missing SSL key should cause an immediate exception."""
-
-        application = Application()
-        module_dir = os.path.dirname(__file__)
-        existing_certificate = os.path.join(module_dir, 'test.crt')
-        existing_key = os.path.join(module_dir, 'test.key')
-
-        self.assertRaises((ValueError, IOError),
-                          HTTPServer, application, ssl_options={
-                              "certfile": "/__mising__.crt",
-        })
-        self.assertRaises((ValueError, IOError),
-                          HTTPServer, application, ssl_options={
-                              "certfile": existing_certificate,
-                              "keyfile": "/__missing__.key"
-        })
-
-        # This actually works because both files exist
-        HTTPServer(application, ssl_options={
-                   "certfile": existing_certificate,
-                   "keyfile": existing_key,
-                   })
-
-
-class MultipartTestHandler(RequestHandler):
-    def post(self):
-        self.finish({"header": self.request.headers["X-Header-Encoding-Test"],
-                     "argument": self.get_argument("argument"),
-                     "filename": self.request.files["files"][0].filename,
-                     "filebody": _unicode(self.request.files["files"][0]["body"]),
-                     })
-
-
-# This test is also called from wsgi_test
-class HTTPConnectionTest(AsyncHTTPTestCase):
-    def get_handlers(self):
-        return [("/multipart", MultipartTestHandler),
-                ("/hello", HelloWorldRequestHandler)]
-
-    def get_app(self):
-        return Application(self.get_handlers())
-
-    def raw_fetch(self, headers, body, newline=b"\r\n"):
-        with closing(IOStream(socket.socket())) as stream:
-            self.io_loop.run_sync(lambda: stream.connect(('127.0.0.1', self.get_http_port())))
-            stream.write(
-                newline.join(headers +
-                             [utf8("Content-Length: %d" % len(body))]) +
-                newline + newline + body)
-            read_stream_body(stream, self.stop)
-            start_line, headers, body = self.wait()
-            return body
-
-    def test_multipart_form(self):
-        # Encodings here are tricky:  Headers are latin1, bodies can be
-        # anything (we use utf8 by default).
-        response = self.raw_fetch([
-            b"POST /multipart HTTP/1.0",
-            b"Content-Type: multipart/form-data; boundary=1234567890",
-            b"X-Header-encoding-test: \xe9",
-        ],
-            b"\r\n".join([
-                b"Content-Disposition: form-data; name=argument",
-                b"",
-                u"\u00e1".encode("utf-8"),
-                b"--1234567890",
-                u'Content-Disposition: form-data; name="files"; filename="\u00f3"'.encode("utf8"),
-                b"",
-                u"\u00fa".encode("utf-8"),
-                b"--1234567890--",
-                b"",
-            ]))
-        data = json_decode(response)
-        self.assertEqual(u"\u00e9", data["header"])
-        self.assertEqual(u"\u00e1", data["argument"])
-        self.assertEqual(u"\u00f3", data["filename"])
-        self.assertEqual(u"\u00fa", data["filebody"])
-
-    def test_newlines(self):
-        # We support both CRLF and bare LF as line separators.
-        for newline in (b"\r\n", b"\n"):
-            response = self.raw_fetch([b"GET /hello HTTP/1.0"], b"",
-                                      newline=newline)
-            self.assertEqual(response, b'Hello world')
-
-    @gen_test
-    def test_100_continue(self):
-        # Run through a 100-continue interaction by hand:
-        # When given Expect: 100-continue, we get a 100 response after the
-        # headers, and then the real response after the body.
-        stream = IOStream(socket.socket())
-        yield stream.connect(("127.0.0.1", self.get_http_port()))
-        yield stream.write(b"\r\n".join([
-            b"POST /hello HTTP/1.1",
-            b"Content-Length: 1024",
-            b"Expect: 100-continue",
-            b"Connection: close",
-            b"\r\n"]))
-        data = yield stream.read_until(b"\r\n\r\n")
-        self.assertTrue(data.startswith(b"HTTP/1.1 100 "), data)
-        stream.write(b"a" * 1024)
-        first_line = yield stream.read_until(b"\r\n")
-        self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line)
-        header_data = yield stream.read_until(b"\r\n\r\n")
-        headers = HTTPHeaders.parse(native_str(header_data.decode('latin1')))
-        body = yield stream.read_bytes(int(headers["Content-Length"]))
-        self.assertEqual(body, b"Got 1024 bytes in POST")
-        stream.close()
-
-
-class EchoHandler(RequestHandler):
-    def get(self):
-        self.write(recursive_unicode(self.request.arguments))
-
-    def post(self):
-        self.write(recursive_unicode(self.request.arguments))
-
-
-class TypeCheckHandler(RequestHandler):
-    def prepare(self):
-        self.errors = {}
-        fields = [
-            ('method', str),
-            ('uri', str),
-            ('version', str),
-            ('remote_ip', str),
-            ('protocol', str),
-            ('host', str),
-            ('path', str),
-            ('query', str),
-        ]
-        for field, expected_type in fields:
-            self.check_type(field, getattr(self.request, field), expected_type)
-
-        self.check_type('header_key', list(self.request.headers.keys())[0], str)
-        self.check_type('header_value', list(self.request.headers.values())[0], str)
-
-        self.check_type('cookie_key', list(self.request.cookies.keys())[0], str)
-        self.check_type('cookie_value', list(self.request.cookies.values())[0].value, str)
-        # secure cookies
-
-        self.check_type('arg_key', list(self.request.arguments.keys())[0], str)
-        self.check_type('arg_value', list(self.request.arguments.values())[0][0], bytes)
-
-    def post(self):
-        self.check_type('body', self.request.body, bytes)
-        self.write(self.errors)
-
-    def get(self):
-        self.write(self.errors)
-
-    def check_type(self, name, obj, expected_type):
-        actual_type = type(obj)
-        if expected_type != actual_type:
-            self.errors[name] = "expected %s, got %s" % (expected_type,
-                                                         actual_type)
-
-
-class HTTPServerTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([("/echo", EchoHandler),
-                            ("/typecheck", TypeCheckHandler),
-                            ("//doubleslash", EchoHandler),
-                            ])
-
-    def test_query_string_encoding(self):
-        response = self.fetch("/echo?foo=%C3%A9")
-        data = json_decode(response.body)
-        self.assertEqual(data, {u"foo": [u"\u00e9"]})
-
-    def test_empty_query_string(self):
-        response = self.fetch("/echo?foo=&foo=")
-        data = json_decode(response.body)
-        self.assertEqual(data, {u"foo": [u"", u""]})
-
-    def test_empty_post_parameters(self):
-        response = self.fetch("/echo", method="POST", body="foo=&bar=")
-        data = json_decode(response.body)
-        self.assertEqual(data, {u"foo": [u""], u"bar": [u""]})
-
-    def test_types(self):
-        headers = {"Cookie": "foo=bar"}
-        response = self.fetch("/typecheck?foo=bar", headers=headers)
-        data = json_decode(response.body)
-        self.assertEqual(data, {})
-
-        response = self.fetch("/typecheck", method="POST", body="foo=bar", headers=headers)
-        data = json_decode(response.body)
-        self.assertEqual(data, {})
-
-    def test_double_slash(self):
-        # urlparse.urlsplit (which tornado.httpserver used to use
-        # incorrectly) would parse paths beginning with "//" as
-        # protocol-relative urls.
-        response = self.fetch("//doubleslash")
-        self.assertEqual(200, response.code)
-        self.assertEqual(json_decode(response.body), {})
-
-    def test_malformed_body(self):
-        # parse_qs is pretty forgiving, but it will fail on python 3
-        # if the data is not utf8.  On python 2 parse_qs will work,
-        # but then the recursive_unicode call in EchoHandler will
-        # fail.
-        if str is bytes:
-            return
-        with ExpectLog(gen_log, 'Invalid x-www-form-urlencoded body'):
-            response = self.fetch(
-                '/echo', method="POST",
-                headers={'Content-Type': 'application/x-www-form-urlencoded'},
-                body=b'\xe9')
-        self.assertEqual(200, response.code)
-        self.assertEqual(b'{}', response.body)
-
-
-class HTTPServerRawTest(AsyncHTTPTestCase):
-    def get_app(self):
-        return Application([
-            ('/echo', EchoHandler),
-        ])
-
-    def setUp(self):
-        super(HTTPServerRawTest, self).setUp()
-        self.stream = IOStream(socket.socket())
-        self.io_loop.run_sync(lambda: self.stream.connect(('127.0.0.1', self.get_http_port())))
-
-    def tearDown(self):
-        self.stream.close()
-        super(HTTPServerRawTest, self).tearDown()
-
-    def test_empty_request(self):
-        self.stream.close()
-        self.io_loop.add_timeout(datetime.timedelta(seconds=0.001), self.stop)
-        self.wait()
-
-    def test_malformed_first_line_response(self):
-        with ExpectLog(gen_log, '.*Malformed HTTP request line'):
-            self.stream.write(b'asdf\r\n\r\n')
-            read_stream_body(self.stream, self.stop)
-            start_line, headers, response = self.wait()
-            self.assertEqual('HTTP/1.1', start_line.version)
-            self.assertEqual(400, start_line.code)
-            self.assertEqual('Bad Request', start_line.reason)
-
-    def test_malformed_first_line_log(self):
-        with ExpectLog(gen_log, '.*Malformed HTTP request line'):
-            self.stream.write(b'asdf\r\n\r\n')
-            # TODO: need an async version of ExpectLog so we don't need
-            # hard-coded timeouts here.
-            self.io_loop.add_timeout(datetime.timedelta(seconds=0.05),
-                                     self.stop)
-            self.wait()
-
-    def test_malformed_headers(self):
-        with ExpectLog(gen_log, '.*Malformed HTTP message.*no colon in header line'):
-            self.stream.write(b'GET / HTTP/1.0\r\nasdf\r\n\r\n')
-            self.io_loop.add_timeout(datetime.timedelta(seconds=0.05),
-                                     self.stop)
-            self.wait()
-
-    def test_chunked_request_body(self):
-        # Chunked requests are not widely supported and we don't have a way
-        # to generate them in AsyncHTTPClient, but HTTPServer will read them.
-        self.stream.write(b"""\
-POST /echo HTTP/1.1
-Transfer-Encoding: chunked
-Content-Type: application/x-www-form-urlencoded
-
-4
-foo=
-3
-bar
-0
-
-""".replace(b"\n", b"\r\n"))
-        read_stream_body(self.stream, self.stop)
-        start_line, headers, response = self.wait()
-        self.assertEqual(json_decode(response), {u'foo': [u'bar']})
-
-    def test_chunked_request_uppercase(self):
-        # As per RFC 2616 section 3.6, "Transfer-Encoding" header's value is
-        # case-insensitive.
-        self.stream.write(b"""\
-POST /echo HTTP/1.1
-Transfer-Encoding: Chunked
-Content-Type: application/x-www-form-urlencoded
-
-4
-foo=
-3
-bar
-0
-
-""".replace(b"\n", b"\r\n"))
-        read_stream_body(self.stream, self.stop)
-        start_line, headers, response = self.wait()
-        self.assertEqual(json_decode(response), {u'foo': [u'bar']})
-
-    @gen_test
-    def test_invalid_content_length(self):
-        with ExpectLog(gen_log, '.*Only integer Content-Length is allowed'):
-            self.stream.write(b"""\
-POST /echo HTTP/1.1
-Content-Length: foo
-
-bar
-
-""".replace(b"\n", b"\r\n"))
-            yield self.stream.read_until_close()
-
-
-class XHeaderTest(HandlerBaseTestCase):
-    class Handler(RequestHandler):
-        def get(self):
-            self.set_header('request-version', self.request.version)
-            self.write(dict(remote_ip=self.request.remote_ip,
-                            remote_protocol=self.request.protocol))
-
-    def get_httpserver_options(self):
-        return dict(xheaders=True, trusted_downstream=['5.5.5.5'])
-
-    def test_ip_headers(self):
-        self.assertEqual(self.fetch_json("/")["remote_ip"], "127.0.0.1")
-
-        valid_ipv4 = {"X-Real-IP": "4.4.4.4"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv4)["remote_ip"],
-            "4.4.4.4")
-
-        valid_ipv4_list = {"X-Forwarded-For": "127.0.0.1, 4.4.4.4"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv4_list)["remote_ip"],
-            "4.4.4.4")
-
-        valid_ipv6 = {"X-Real-IP": "2620:0:1cfe:face:b00c::3"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv6)["remote_ip"],
-            "2620:0:1cfe:face:b00c::3")
-
-        valid_ipv6_list = {"X-Forwarded-For": "::1, 2620:0:1cfe:face:b00c::3"}
-        self.assertEqual(
-            self.fetch_json("/", headers=valid_ipv6_list)["remote_ip"],
-            "2620:0:1cfe:face:b00c::3")
-
-        invalid_chars = {"X-Real-IP": "4.4.4.4
-
-'
-                       for p in paths)
-
-    def render_embed_js(self, js_embed):
-        """Default method used to render the final embedded js for the
-        rendered webpage.
-
-        Override this method in a sub-classed controller to change the output.
-        """
-        return b''
-
-    def render_linked_css(self, css_files):
-        """Default method used to render the final css links for the
-        rendered webpage.
-
-        Override this method in a sub-classed controller to change the output.
-        """
-        paths = []
-        unique_paths = set()
-
-        for path in css_files:
-            if not is_absolute(path):
-                path = self.static_url(path)
-            if path not in unique_paths:
-                paths.append(path)
-                unique_paths.add(path)
-
-        return ''.join(''
-                       for p in paths)
-
-    def render_embed_css(self, css_embed):
-        """Default method used to render the final embedded css for the
-        rendered webpage.
-
-        Override this method in a sub-classed controller to change the output.
-        """
-        return b''
-
-    def render_string(self, template_name, **kwargs):
-        """Generate the given template with the given arguments.
-
-        We return the generated byte string (in utf8). To generate and
-        write a template as a response, use render() above.
-        """
-        # If no template_path is specified, use the path of the calling file
-        template_path = self.get_template_path()
-        if not template_path:
-            frame = sys._getframe(0)
-            web_file = frame.f_code.co_filename
-            while frame.f_code.co_filename == web_file:
-                frame = frame.f_back
-            template_path = os.path.dirname(frame.f_code.co_filename)
-        with RequestHandler._template_loader_lock:
-            if template_path not in RequestHandler._template_loaders:
-                loader = self.create_template_loader(template_path)
-                RequestHandler._template_loaders[template_path] = loader
-            else:
-                loader = RequestHandler._template_loaders[template_path]
-        t = loader.load(template_name)
-        namespace = self.get_template_namespace()
-        namespace.update(kwargs)
-        return t.generate(**namespace)
-
-    def get_template_namespace(self):
-        """Returns a dictionary to be used as the default template namespace.
-
-        May be overridden by subclasses to add or modify values.
-
-        The results of this method will be combined with additional
-        defaults in the `tornado.template` module and keyword arguments
-        to `render` or `render_string`.
-        """
-        namespace = dict(
-            handler=self,
-            request=self.request,
-            current_user=self.current_user,
-            locale=self.locale,
-            _=self.locale.translate,
-            pgettext=self.locale.pgettext,
-            static_url=self.static_url,
-            xsrf_form_html=self.xsrf_form_html,
-            reverse_url=self.reverse_url
-        )
-        namespace.update(self.ui)
-        return namespace
-
-    def create_template_loader(self, template_path):
-        """Returns a new template loader for the given path.
-
-        May be overridden by subclasses.  By default returns a
-        directory-based loader on the given path, using the
-        ``autoescape`` and ``template_whitespace`` application
-        settings.  If a ``template_loader`` application setting is
-        supplied, uses that instead.
-        """
-        settings = self.application.settings
-        if "template_loader" in settings:
-            return settings["template_loader"]
-        kwargs = {}
-        if "autoescape" in settings:
-            # autoescape=None means "no escaping", so we have to be sure
-            # to only pass this kwarg if the user asked for it.
-            kwargs["autoescape"] = settings["autoescape"]
-        if "template_whitespace" in settings:
-            kwargs["whitespace"] = settings["template_whitespace"]
-        return template.Loader(template_path, **kwargs)
-
-    def flush(self, include_footers=False, callback=None):
-        """Flushes the current output buffer to the network.
-
-        The ``callback`` argument, if given, can be used for flow control:
-        it will be run when all flushed data has been written to the socket.
-        Note that only one flush callback can be outstanding at a time;
-        if another flush occurs before the previous flush's callback
-        has been run, the previous callback will be discarded.
-
-        .. versionchanged:: 4.0
-           Now returns a `.Future` if no callback is given.
-
-        .. deprecated:: 5.1
-
-           The ``callback`` argument is deprecated and will be removed in
-           Tornado 6.0.
-        """
-        chunk = b"".join(self._write_buffer)
-        self._write_buffer = []
-        if not self._headers_written:
-            self._headers_written = True
-            for transform in self._transforms:
-                self._status_code, self._headers, chunk = \
-                    transform.transform_first_chunk(
-                        self._status_code, self._headers,
-                        chunk, include_footers)
-            # Ignore the chunk and only write the headers for HEAD requests
-            if self.request.method == "HEAD":
-                chunk = None
-
-            # Finalize the cookie headers (which have been stored in a side
-            # object so an outgoing cookie could be overwritten before it
-            # is sent).
-            if hasattr(self, "_new_cookie"):
-                for cookie in self._new_cookie.values():
-                    self.add_header("Set-Cookie", cookie.OutputString(None))
-
-            start_line = httputil.ResponseStartLine('',
-                                                    self._status_code,
-                                                    self._reason)
-            return self.request.connection.write_headers(
-                start_line, self._headers, chunk, callback=callback)
-        else:
-            for transform in self._transforms:
-                chunk = transform.transform_chunk(chunk, include_footers)
-            # Ignore the chunk and only write the headers for HEAD requests
-            if self.request.method != "HEAD":
-                return self.request.connection.write(chunk, callback=callback)
-            else:
-                future = Future()
-                future.set_result(None)
-                return future
-
-    def finish(self, chunk=None):
-        """Finishes this response, ending the HTTP request.
-
-        Passing a ``chunk`` to ``finish()`` is equivalent to passing that
-        chunk to ``write()`` and then calling ``finish()`` with no arguments.
-
-        Returns a `.Future` which may optionally be awaited to track the sending
-        of the response to the client. This `.Future` resolves when all the response
-        data has been sent, and raises an error if the connection is closed before all
-        data can be sent.
-
-        .. versionchanged:: 5.1
-
-           Now returns a `.Future` instead of ``None``.
-        """
-        if self._finished:
-            raise RuntimeError("finish() called twice")
-
-        if chunk is not None:
-            self.write(chunk)
-
-        # Automatically support ETags and add the Content-Length header if
-        # we have not flushed any content yet.
-        if not self._headers_written:
-            if (self._status_code == 200 and
-                self.request.method in ("GET", "HEAD") and
-                    "Etag" not in self._headers):
-                self.set_etag_header()
-                if self.check_etag_header():
-                    self._write_buffer = []
-                    self.set_status(304)
-            if (self._status_code in (204, 304) or
-                    (self._status_code >= 100 and self._status_code < 200)):
-                assert not self._write_buffer, "Cannot send body with %s" % self._status_code
-                self._clear_headers_for_304()
-            elif "Content-Length" not in self._headers:
-                content_length = sum(len(part) for part in self._write_buffer)
-                self.set_header("Content-Length", content_length)
-
-        if hasattr(self.request, "connection"):
-            # Now that the request is finished, clear the callback we
-            # set on the HTTPConnection (which would otherwise prevent the
-            # garbage collection of the RequestHandler when there
-            # are keepalive connections)
-            self.request.connection.set_close_callback(None)
-
-        future = self.flush(include_footers=True)
-        self.request.connection.finish()
-        self._log()
-        self._finished = True
-        self.on_finish()
-        self._break_cycles()
-        return future
-
-    def detach(self):
-        """Take control of the underlying stream.
-
-        Returns the underlying `.IOStream` object and stops all
-        further HTTP processing. Intended for implementing protocols
-        like websockets that tunnel over an HTTP handshake.
-
-        This method is only supported when HTTP/1.1 is used.
-
-        .. versionadded:: 5.1
-        """
-        self._finished = True
-        return self.request.connection.detach()
-
-    def _break_cycles(self):
-        # Break up a reference cycle between this handler and the
-        # _ui_module closures to allow for faster GC on CPython.
-        self.ui = None
-
-    def send_error(self, status_code=500, **kwargs):
-        """Sends the given HTTP error code to the browser.
-
-        If `flush()` has already been called, it is not possible to send
-        an error, so this method will simply terminate the response.
-        If output has been written but not yet flushed, it will be discarded
-        and replaced with the error page.
-
-        Override `write_error()` to customize the error page that is returned.
-        Additional keyword arguments are passed through to `write_error`.
-        """
-        if self._headers_written:
-            gen_log.error("Cannot send error response after headers written")
-            if not self._finished:
-                # If we get an error between writing headers and finishing,
-                # we are unlikely to be able to finish due to a
-                # Content-Length mismatch. Try anyway to release the
-                # socket.
-                try:
-                    self.finish()
-                except Exception:
-                    gen_log.error("Failed to flush partial response",
-                                  exc_info=True)
-            return
-        self.clear()
-
-        reason = kwargs.get('reason')
-        if 'exc_info' in kwargs:
-            exception = kwargs['exc_info'][1]
-            if isinstance(exception, HTTPError) and exception.reason:
-                reason = exception.reason
-        self.set_status(status_code, reason=reason)
-        try:
-            self.write_error(status_code, **kwargs)
-        except Exception:
-            app_log.error("Uncaught exception in write_error", exc_info=True)
-        if not self._finished:
-            self.finish()
-
-    def write_error(self, status_code, **kwargs):
-        """Override to implement custom error pages.
-
-        ``write_error`` may call `write`, `render`, `set_header`, etc
-        to produce output as usual.
-
-        If this error was caused by an uncaught exception (including
-        HTTPError), an ``exc_info`` triple will be available as
-        ``kwargs["exc_info"]``.  Note that this exception may not be
-        the "current" exception for purposes of methods like
-        ``sys.exc_info()`` or ``traceback.format_exc``.
-        """
-        if self.settings.get("serve_traceback") and "exc_info" in kwargs:
-            # in debug mode, try to send a traceback
-            self.set_header('Content-Type', 'text/plain')
-            for line in traceback.format_exception(*kwargs["exc_info"]):
-                self.write(line)
-            self.finish()
-        else:
-            self.finish("%(code)d: %(message)s"
-                        "%(code)d: %(message)s" % {
-                            "code": status_code,
-                            "message": self._reason,
-                        })
-
-    @property
-    def locale(self):
-        """The locale for the current session.
-
-        Determined by either `get_user_locale`, which you can override to
-        set the locale based on, e.g., a user preference stored in a
-        database, or `get_browser_locale`, which uses the ``Accept-Language``
-        header.
-
-        .. versionchanged: 4.1
-           Added a property setter.
-        """
-        if not hasattr(self, "_locale"):
-            self._locale = self.get_user_locale()
-            if not self._locale:
-                self._locale = self.get_browser_locale()
-                assert self._locale
-        return self._locale
-
-    @locale.setter
-    def locale(self, value):
-        self._locale = value
-
-    def get_user_locale(self):
-        """Override to determine the locale from the authenticated user.
-
-        If None is returned, we fall back to `get_browser_locale()`.
-
-        This method should return a `tornado.locale.Locale` object,
-        most likely obtained via a call like ``tornado.locale.get("en")``
-        """
-        return None
-
-    def get_browser_locale(self, default="en_US"):
-        """Determines the user's locale from ``Accept-Language`` header.
-
-        See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
-        """
-        if "Accept-Language" in self.request.headers:
-            languages = self.request.headers["Accept-Language"].split(",")
-            locales = []
-            for language in languages:
-                parts = language.strip().split(";")
-                if len(parts) > 1 and parts[1].startswith("q="):
-                    try:
-                        score = float(parts[1][2:])
-                    except (ValueError, TypeError):
-                        score = 0.0
-                else:
-                    score = 1.0
-                locales.append((parts[0], score))
-            if locales:
-                locales.sort(key=lambda pair: pair[1], reverse=True)
-                codes = [l[0] for l in locales]
-                return locale.get(*codes)
-        return locale.get(default)
-
-    @property
-    def current_user(self):
-        """The authenticated user for this request.
-
-        This is set in one of two ways:
-
-        * A subclass may override `get_current_user()`, which will be called
-          automatically the first time ``self.current_user`` is accessed.
-          `get_current_user()` will only be called once per request,
-          and is cached for future access::
-
-              def get_current_user(self):
-                  user_cookie = self.get_secure_cookie("user")
-                  if user_cookie:
-                      return json.loads(user_cookie)
-                  return None
-
-        * It may be set as a normal variable, typically from an overridden
-          `prepare()`::
-
-              @gen.coroutine
-              def prepare(self):
-                  user_id_cookie = self.get_secure_cookie("user_id")
-                  if user_id_cookie:
-                      self.current_user = yield load_user(user_id_cookie)
-
-        Note that `prepare()` may be a coroutine while `get_current_user()`
-        may not, so the latter form is necessary if loading the user requires
-        asynchronous operations.
-
-        The user object may be any type of the application's choosing.
-        """
-        if not hasattr(self, "_current_user"):
-            self._current_user = self.get_current_user()
-        return self._current_user
-
-    @current_user.setter
-    def current_user(self, value):
-        self._current_user = value
-
-    def get_current_user(self):
-        """Override to determine the current user from, e.g., a cookie.
-
-        This method may not be a coroutine.
-        """
-        return None
-
-    def get_login_url(self):
-        """Override to customize the login URL based on the request.
-
-        By default, we use the ``login_url`` application setting.
-        """
-        self.require_setting("login_url", "@tornado.web.authenticated")
-        return self.application.settings["login_url"]
-
-    def get_template_path(self):
-        """Override to customize template path for each handler.
-
-        By default, we use the ``template_path`` application setting.
-        Return None to load templates relative to the calling file.
-        """
-        return self.application.settings.get("template_path")
-
-    @property
-    def xsrf_token(self):
-        """The XSRF-prevention token for the current user/session.
-
-        To prevent cross-site request forgery, we set an '_xsrf' cookie
-        and include the same '_xsrf' value as an argument with all POST
-        requests. If the two do not match, we reject the form submission
-        as a potential forgery.
-
-        See http://en.wikipedia.org/wiki/Cross-site_request_forgery
-
-        This property is of type `bytes`, but it contains only ASCII
-        characters. If a character string is required, there is no
-        need to base64-encode it; just decode the byte string as
-        UTF-8.
-
-        .. versionchanged:: 3.2.2
-           The xsrf token will now be have a random mask applied in every
-           request, which makes it safe to include the token in pages
-           that are compressed.  See http://breachattack.com for more
-           information on the issue fixed by this change.  Old (version 1)
-           cookies will be converted to version 2 when this method is called
-           unless the ``xsrf_cookie_version`` `Application` setting is
-           set to 1.
-
-        .. versionchanged:: 4.3
-           The ``xsrf_cookie_kwargs`` `Application` setting may be
-           used to supply additional cookie options (which will be
-           passed directly to `set_cookie`). For example,
-           ``xsrf_cookie_kwargs=dict(httponly=True, secure=True)``
-           will set the ``secure`` and ``httponly`` flags on the
-           ``_xsrf`` cookie.
-        """
-        if not hasattr(self, "_xsrf_token"):
-            version, token, timestamp = self._get_raw_xsrf_token()
-            output_version = self.settings.get("xsrf_cookie_version", 2)
-            cookie_kwargs = self.settings.get("xsrf_cookie_kwargs", {})
-            if output_version == 1:
-                self._xsrf_token = binascii.b2a_hex(token)
-            elif output_version == 2:
-                mask = os.urandom(4)
-                self._xsrf_token = b"|".join([
-                    b"2",
-                    binascii.b2a_hex(mask),
-                    binascii.b2a_hex(_websocket_mask(mask, token)),
-                    utf8(str(int(timestamp)))])
-            else:
-                raise ValueError("unknown xsrf cookie version %d",
-                                 output_version)
-            if version is None:
-                expires_days = 30 if self.current_user else None
-                self.set_cookie("_xsrf", self._xsrf_token,
-                                expires_days=expires_days,
-                                **cookie_kwargs)
-        return self._xsrf_token
-
-    def _get_raw_xsrf_token(self):
-        """Read or generate the xsrf token in its raw form.
-
-        The raw_xsrf_token is a tuple containing:
-
-        * version: the version of the cookie from which this token was read,
-          or None if we generated a new token in this request.
-        * token: the raw token data; random (non-ascii) bytes.
-        * timestamp: the time this token was generated (will not be accurate
-          for version 1 cookies)
-        """
-        if not hasattr(self, '_raw_xsrf_token'):
-            cookie = self.get_cookie("_xsrf")
-            if cookie:
-                version, token, timestamp = self._decode_xsrf_token(cookie)
-            else:
-                version, token, timestamp = None, None, None
-            if token is None:
-                version = None
-                token = os.urandom(16)
-                timestamp = time.time()
-            self._raw_xsrf_token = (version, token, timestamp)
-        return self._raw_xsrf_token
-
-    def _decode_xsrf_token(self, cookie):
-        """Convert a cookie string into a the tuple form returned by
-        _get_raw_xsrf_token.
-        """
-
-        try:
-            m = _signed_value_version_re.match(utf8(cookie))
-
-            if m:
-                version = int(m.group(1))
-                if version == 2:
-                    _, mask, masked_token, timestamp = cookie.split("|")
-
-                    mask = binascii.a2b_hex(utf8(mask))
-                    token = _websocket_mask(
-                        mask, binascii.a2b_hex(utf8(masked_token)))
-                    timestamp = int(timestamp)
-                    return version, token, timestamp
-                else:
-                    # Treat unknown versions as not present instead of failing.
-                    raise Exception("Unknown xsrf cookie version")
-            else:
-                version = 1
-                try:
-                    token = binascii.a2b_hex(utf8(cookie))
-                except (binascii.Error, TypeError):
-                    token = utf8(cookie)
-                # We don't have a usable timestamp in older versions.
-                timestamp = int(time.time())
-                return (version, token, timestamp)
-        except Exception:
-            # Catch exceptions and return nothing instead of failing.
-            gen_log.debug("Uncaught exception in _decode_xsrf_token",
-                          exc_info=True)
-            return None, None, None
-
-    def check_xsrf_cookie(self):
-        """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.
-
-        To prevent cross-site request forgery, we set an ``_xsrf``
-        cookie and include the same value as a non-cookie
-        field with all ``POST`` requests. If the two do not match, we
-        reject the form submission as a potential forgery.
-
-        The ``_xsrf`` value may be set as either a form field named ``_xsrf``
-        or in a custom HTTP header named ``X-XSRFToken`` or ``X-CSRFToken``
-        (the latter is accepted for compatibility with Django).
-
-        See http://en.wikipedia.org/wiki/Cross-site_request_forgery
-
-        Prior to release 1.1.1, this check was ignored if the HTTP header
-        ``X-Requested-With: XMLHTTPRequest`` was present.  This exception
-        has been shown to be insecure and has been removed.  For more
-        information please see
-        http://www.djangoproject.com/weblog/2011/feb/08/security/
-        http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails
-
-        .. versionchanged:: 3.2.2
-           Added support for cookie version 2.  Both versions 1 and 2 are
-           supported.
-        """
-        token = (self.get_argument("_xsrf", None) or
-                 self.request.headers.get("X-Xsrftoken") or
-                 self.request.headers.get("X-Csrftoken"))
-        if not token:
-            raise HTTPError(403, "'_xsrf' argument missing from POST")
-        _, token, _ = self._decode_xsrf_token(token)
-        _, expected_token, _ = self._get_raw_xsrf_token()
-        if not token:
-            raise HTTPError(403, "'_xsrf' argument has invalid format")
-        if not _time_independent_equals(utf8(token), utf8(expected_token)):
-            raise HTTPError(403, "XSRF cookie does not match POST argument")
-
-    def xsrf_form_html(self):
-        """An HTML ```` element to be included with all POST forms.
-
-        It defines the ``_xsrf`` input value, which we check on all POST
-        requests to prevent cross-site request forgery. If you have set
-        the ``xsrf_cookies`` application setting, you must include this
-        HTML within all of your HTML forms.
-
-        In a template, this method should be called with ``{% module
-        xsrf_form_html() %}``
-
-        See `check_xsrf_cookie()` above for more information.
-        """
-        return ''
-
-    def static_url(self, path, include_host=None, **kwargs):
-        """Returns a static URL for the given relative static file path.
-
-        This method requires you set the ``static_path`` setting in your
-        application (which specifies the root directory of your static
-        files).
-
-        This method returns a versioned url (by default appending
-        ``?v=``), which allows the static files to be
-        cached indefinitely.  This can be disabled by passing
-        ``include_version=False`` (in the default implementation;
-        other static file implementations are not required to support
-        this, but they may support other options).
-
-        By default this method returns URLs relative to the current
-        host, but if ``include_host`` is true the URL returned will be
-        absolute.  If this handler has an ``include_host`` attribute,
-        that value will be used as the default for all `static_url`
-        calls that do not pass ``include_host`` as a keyword argument.
-
-        """
-        self.require_setting("static_path", "static_url")
-        get_url = self.settings.get("static_handler_class",
-                                    StaticFileHandler).make_static_url
-
-        if include_host is None:
-            include_host = getattr(self, "include_host", False)
-
-        if include_host:
-            base = self.request.protocol + "://" + self.request.host
-        else:
-            base = ""
-
-        return base + get_url(self.settings, path, **kwargs)
-
-    def require_setting(self, name, feature="this feature"):
-        """Raises an exception if the given app setting is not defined."""
-        if not self.application.settings.get(name):
-            raise Exception("You must define the '%s' setting in your "
-                            "application to use %s" % (name, feature))
-
-    def reverse_url(self, name, *args):
-        """Alias for `Application.reverse_url`."""
-        return self.application.reverse_url(name, *args)
-
-    def compute_etag(self):
-        """Computes the etag header to be used for this request.
-
-        By default uses a hash of the content written so far.
-
-        May be overridden to provide custom etag implementations,
-        or may return None to disable tornado's default etag support.
-        """
-        hasher = hashlib.sha1()
-        for part in self._write_buffer:
-            hasher.update(part)
-        return '"%s"' % hasher.hexdigest()
-
-    def set_etag_header(self):
-        """Sets the response's Etag header using ``self.compute_etag()``.
-
-        Note: no header will be set if ``compute_etag()`` returns ``None``.
-
-        This method is called automatically when the request is finished.
-        """
-        etag = self.compute_etag()
-        if etag is not None:
-            self.set_header("Etag", etag)
-
-    def check_etag_header(self):
-        """Checks the ``Etag`` header against requests's ``If-None-Match``.
-
-        Returns ``True`` if the request's Etag matches and a 304 should be
-        returned. For example::
-
-            self.set_etag_header()
-            if self.check_etag_header():
-                self.set_status(304)
-                return
-
-        This method is called automatically when the request is finished,
-        but may be called earlier for applications that override
-        `compute_etag` and want to do an early check for ``If-None-Match``
-        before completing the request.  The ``Etag`` header should be set
-        (perhaps with `set_etag_header`) before calling this method.
-        """
-        computed_etag = utf8(self._headers.get("Etag", ""))
-        # Find all weak and strong etag values from If-None-Match header
-        # because RFC 7232 allows multiple etag values in a single header.
-        etags = re.findall(
-            br'\*|(?:W/)?"[^"]*"',
-            utf8(self.request.headers.get("If-None-Match", ""))
-        )
-        if not computed_etag or not etags:
-            return False
-
-        match = False
-        if etags[0] == b'*':
-            match = True
-        else:
-            # Use a weak comparison when comparing entity-tags.
-            def val(x):
-                return x[2:] if x.startswith(b'W/') else x
-
-            for etag in etags:
-                if val(etag) == val(computed_etag):
-                    match = True
-                    break
-        return match
-
-    def _stack_context_handle_exception(self, type, value, traceback):
-        try:
-            # For historical reasons _handle_request_exception only takes
-            # the exception value instead of the full triple,
-            # so re-raise the exception to ensure that it's in
-            # sys.exc_info()
-            raise_exc_info((type, value, traceback))
-        except Exception:
-            self._handle_request_exception(value)
-        return True
-
-    @gen.coroutine
-    def _execute(self, transforms, *args, **kwargs):
-        """Executes this request with the given output transforms."""
-        self._transforms = transforms
-        try:
-            if self.request.method not in self.SUPPORTED_METHODS:
-                raise HTTPError(405)
-            self.path_args = [self.decode_argument(arg) for arg in args]
-            self.path_kwargs = dict((k, self.decode_argument(v, name=k))
-                                    for (k, v) in kwargs.items())
-            # If XSRF cookies are turned on, reject form submissions without
-            # the proper cookie
-            if self.request.method not in ("GET", "HEAD", "OPTIONS") and \
-                    self.application.settings.get("xsrf_cookies"):
-                self.check_xsrf_cookie()
-
-            result = self.prepare()
-            if result is not None:
-                result = yield result
-            if self._prepared_future is not None:
-                # Tell the Application we've finished with prepare()
-                # and are ready for the body to arrive.
-                future_set_result_unless_cancelled(self._prepared_future, None)
-            if self._finished:
-                return
-
-            if _has_stream_request_body(self.__class__):
-                # In streaming mode request.body is a Future that signals
-                # the body has been completely received.  The Future has no
-                # result; the data has been passed to self.data_received
-                # instead.
-                try:
-                    yield self.request.body
-                except iostream.StreamClosedError:
-                    return
-
-            method = getattr(self, self.request.method.lower())
-            result = method(*self.path_args, **self.path_kwargs)
-            if result is not None:
-                result = yield result
-            if self._auto_finish and not self._finished:
-                self.finish()
-        except Exception as e:
-            try:
-                self._handle_request_exception(e)
-            except Exception:
-                app_log.error("Exception in exception handler", exc_info=True)
-            finally:
-                # Unset result to avoid circular references
-                result = None
-            if (self._prepared_future is not None and
-                    not self._prepared_future.done()):
-                # In case we failed before setting _prepared_future, do it
-                # now (to unblock the HTTP server).  Note that this is not
-                # in a finally block to avoid GC issues prior to Python 3.4.
-                self._prepared_future.set_result(None)
-
-    def data_received(self, chunk):
-        """Implement this method to handle streamed request data.
-
-        Requires the `.stream_request_body` decorator.
-        """
-        raise NotImplementedError()
-
-    def _log(self):
-        """Logs the current request.
-
-        Sort of deprecated since this functionality was moved to the
-        Application, but left in place for the benefit of existing apps
-        that have overridden this method.
-        """
-        self.application.log_request(self)
-
-    def _request_summary(self):
-        return "%s %s (%s)" % (self.request.method, self.request.uri,
-                               self.request.remote_ip)
-
-    def _handle_request_exception(self, e):
-        if isinstance(e, Finish):
-            # Not an error; just finish the request without logging.
-            if not self._finished:
-                self.finish(*e.args)
-            return
-        try:
-            self.log_exception(*sys.exc_info())
-        except Exception:
-            # An error here should still get a best-effort send_error()
-            # to avoid leaking the connection.
-            app_log.error("Error in exception logger", exc_info=True)
-        if self._finished:
-            # Extra errors after the request has been finished should
-            # be logged, but there is no reason to continue to try and
-            # send a response.
-            return
-        if isinstance(e, HTTPError):
-            self.send_error(e.status_code, exc_info=sys.exc_info())
-        else:
-            self.send_error(500, exc_info=sys.exc_info())
-
-    def log_exception(self, typ, value, tb):
-        """Override to customize logging of uncaught exceptions.
-
-        By default logs instances of `HTTPError` as warnings without
-        stack traces (on the ``tornado.general`` logger), and all
-        other exceptions as errors with stack traces (on the
-        ``tornado.application`` logger).
-
-        .. versionadded:: 3.1
-        """
-        if isinstance(value, HTTPError):
-            if value.log_message:
-                format = "%d %s: " + value.log_message
-                args = ([value.status_code, self._request_summary()] +
-                        list(value.args))
-                gen_log.warning(format, *args)
-        else:
-            app_log.error("Uncaught exception %s\n%r", self._request_summary(),
-                          self.request, exc_info=(typ, value, tb))
-
-    def _ui_module(self, name, module):
-        def render(*args, **kwargs):
-            if not hasattr(self, "_active_modules"):
-                self._active_modules = {}
-            if name not in self._active_modules:
-                self._active_modules[name] = module(self)
-            rendered = self._active_modules[name].render(*args, **kwargs)
-            return rendered
-        return render
-
-    def _ui_method(self, method):
-        return lambda *args, **kwargs: method(self, *args, **kwargs)
-
-    def _clear_headers_for_304(self):
-        # 304 responses should not contain entity headers (defined in
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.1)
-        # not explicitly allowed by
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
-        headers = ["Allow", "Content-Encoding", "Content-Language",
-                   "Content-Length", "Content-MD5", "Content-Range",
-                   "Content-Type", "Last-Modified"]
-        for h in headers:
-            self.clear_header(h)
-
-
-def asynchronous(method):
-    """Wrap request handler methods with this if they are asynchronous.
-
-    This decorator is for callback-style asynchronous methods; for
-    coroutines, use the ``@gen.coroutine`` decorator without
-    ``@asynchronous``. (It is legal for legacy reasons to use the two
-    decorators together provided ``@asynchronous`` is first, but
-    ``@asynchronous`` will be ignored in this case)
-
-    This decorator should only be applied to the :ref:`HTTP verb
-    methods `; its behavior is undefined for any other method.
-    This decorator does not *make* a method asynchronous; it tells
-    the framework that the method *is* asynchronous.  For this decorator
-    to be useful the method must (at least sometimes) do something
-    asynchronous.
-
-    If this decorator is given, the response is not finished when the
-    method returns. It is up to the request handler to call
-    `self.finish() ` to finish the HTTP
-    request. Without this decorator, the request is automatically
-    finished when the ``get()`` or ``post()`` method returns. Example:
-
-    .. testcode::
-
-       class MyRequestHandler(RequestHandler):
-           @asynchronous
-           def get(self):
-              http = httpclient.AsyncHTTPClient()
-              http.fetch("http://friendfeed.com/", self._on_download)
-
-           def _on_download(self, response):
-              self.write("Downloaded!")
-              self.finish()
-
-    .. testoutput::
-       :hide:
-
-    .. versionchanged:: 3.1
-       The ability to use ``@gen.coroutine`` without ``@asynchronous``.
-
-    .. versionchanged:: 4.3 Returning anything but ``None`` or a
-       yieldable object from a method decorated with ``@asynchronous``
-       is an error. Such return values were previously ignored silently.
-
-    .. deprecated:: 5.1
-
-       This decorator is deprecated and will be removed in Tornado 6.0.
-       Use coroutines instead.
-    """
-    warnings.warn("@asynchronous is deprecated, use coroutines instead",
-                  DeprecationWarning)
-    # Delay the IOLoop import because it's not available on app engine.
-    from tornado.ioloop import IOLoop
-
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        self._auto_finish = False
-        with stack_context.ExceptionStackContext(
-                self._stack_context_handle_exception, delay_warning=True):
-            result = method(self, *args, **kwargs)
-            if result is not None:
-                result = gen.convert_yielded(result)
-
-                # If @asynchronous is used with @gen.coroutine, (but
-                # not @gen.engine), we can automatically finish the
-                # request when the future resolves.  Additionally,
-                # the Future will swallow any exceptions so we need
-                # to throw them back out to the stack context to finish
-                # the request.
-                def future_complete(f):
-                    f.result()
-                    if not self._finished:
-                        self.finish()
-                IOLoop.current().add_future(result, future_complete)
-                # Once we have done this, hide the Future from our
-                # caller (i.e. RequestHandler._when_complete), which
-                # would otherwise set up its own callback and
-                # exception handler (resulting in exceptions being
-                # logged twice).
-                return None
-            return result
-    return wrapper
-
-
-def stream_request_body(cls):
-    """Apply to `RequestHandler` subclasses to enable streaming body support.
-
-    This decorator implies the following changes:
-
-    * `.HTTPServerRequest.body` is undefined, and body arguments will not
-      be included in `RequestHandler.get_argument`.
-    * `RequestHandler.prepare` is called when the request headers have been
-      read instead of after the entire body has been read.
-    * The subclass must define a method ``data_received(self, data):``, which
-      will be called zero or more times as data is available.  Note that
-      if the request has an empty body, ``data_received`` may not be called.
-    * ``prepare`` and ``data_received`` may return Futures (such as via
-      ``@gen.coroutine``, in which case the next method will not be called
-      until those futures have completed.
-    * The regular HTTP method (``post``, ``put``, etc) will be called after
-      the entire body has been read.
-
-    See the `file receiver demo `_
-    for example usage.
-    """  # noqa: E501
-    if not issubclass(cls, RequestHandler):
-        raise TypeError("expected subclass of RequestHandler, got %r", cls)
-    cls._stream_request_body = True
-    return cls
-
-
-def _has_stream_request_body(cls):
-    if not issubclass(cls, RequestHandler):
-        raise TypeError("expected subclass of RequestHandler, got %r", cls)
-    return getattr(cls, '_stream_request_body', False)
-
-
-def removeslash(method):
-    """Use this decorator to remove trailing slashes from the request path.
-
-    For example, a request to ``/foo/`` would redirect to ``/foo`` with this
-    decorator. Your request handler mapping should use a regular expression
-    like ``r'/foo/*'`` in conjunction with using the decorator.
-    """
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if self.request.path.endswith("/"):
-            if self.request.method in ("GET", "HEAD"):
-                uri = self.request.path.rstrip("/")
-                if uri:  # don't try to redirect '/' to ''
-                    if self.request.query:
-                        uri += "?" + self.request.query
-                    self.redirect(uri, permanent=True)
-                    return
-            else:
-                raise HTTPError(404)
-        return method(self, *args, **kwargs)
-    return wrapper
-
-
-def addslash(method):
-    """Use this decorator to add a missing trailing slash to the request path.
-
-    For example, a request to ``/foo`` would redirect to ``/foo/`` with this
-    decorator. Your request handler mapping should use a regular expression
-    like ``r'/foo/?'`` in conjunction with using the decorator.
-    """
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if not self.request.path.endswith("/"):
-            if self.request.method in ("GET", "HEAD"):
-                uri = self.request.path + "/"
-                if self.request.query:
-                    uri += "?" + self.request.query
-                self.redirect(uri, permanent=True)
-                return
-            raise HTTPError(404)
-        return method(self, *args, **kwargs)
-    return wrapper
-
-
-class _ApplicationRouter(ReversibleRuleRouter):
-    """Routing implementation used internally by `Application`.
-
-    Provides a binding between `Application` and `RequestHandler`.
-    This implementation extends `~.routing.ReversibleRuleRouter` in a couple of ways:
-        * it allows to use `RequestHandler` subclasses as `~.routing.Rule` target and
-        * it allows to use a list/tuple of rules as `~.routing.Rule` target.
-        ``process_rule`` implementation will substitute this list with an appropriate
-        `_ApplicationRouter` instance.
-    """
-
-    def __init__(self, application, rules=None):
-        assert isinstance(application, Application)
-        self.application = application
-        super(_ApplicationRouter, self).__init__(rules)
-
-    def process_rule(self, rule):
-        rule = super(_ApplicationRouter, self).process_rule(rule)
-
-        if isinstance(rule.target, (list, tuple)):
-            rule.target = _ApplicationRouter(self.application, rule.target)
-
-        return rule
-
-    def get_target_delegate(self, target, request, **target_params):
-        if isclass(target) and issubclass(target, RequestHandler):
-            return self.application.get_handler_delegate(request, target, **target_params)
-
-        return super(_ApplicationRouter, self).get_target_delegate(target, request, **target_params)
-
-
-class Application(ReversibleRouter):
-    r"""A collection of request handlers that make up a web application.
-
-    Instances of this class are callable and can be passed directly to
-    HTTPServer to serve the application::
-
-        application = web.Application([
-            (r"/", MainPageHandler),
-        ])
-        http_server = httpserver.HTTPServer(application)
-        http_server.listen(8080)
-        ioloop.IOLoop.current().start()
-
-    The constructor for this class takes in a list of `~.routing.Rule`
-    objects or tuples of values corresponding to the arguments of
-    `~.routing.Rule` constructor: ``(matcher, target, [target_kwargs], [name])``,
-    the values in square brackets being optional. The default matcher is
-    `~.routing.PathMatches`, so ``(regexp, target)`` tuples can also be used
-    instead of ``(PathMatches(regexp), target)``.
-
-    A common routing target is a `RequestHandler` subclass, but you can also
-    use lists of rules as a target, which create a nested routing configuration::
-
-        application = web.Application([
-            (HostMatches("example.com"), [
-                (r"/", MainPageHandler),
-                (r"/feed", FeedHandler),
-            ]),
-        ])
-
-    In addition to this you can use nested `~.routing.Router` instances,
-    `~.httputil.HTTPMessageDelegate` subclasses and callables as routing targets
-    (see `~.routing` module docs for more information).
-
-    When we receive requests, we iterate over the list in order and
-    instantiate an instance of the first request class whose regexp
-    matches the request path. The request class can be specified as
-    either a class object or a (fully-qualified) name.
-
-    A dictionary may be passed as the third element (``target_kwargs``)
-    of the tuple, which will be used as keyword arguments to the handler's
-    constructor and `~RequestHandler.initialize` method. This pattern
-    is used for the `StaticFileHandler` in this example (note that a
-    `StaticFileHandler` can be installed automatically with the
-    static_path setting described below)::
-
-        application = web.Application([
-            (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
-        ])
-
-    We support virtual hosts with the `add_handlers` method, which takes in
-    a host regular expression as the first argument::
-
-        application.add_handlers(r"www\.myhost\.com", [
-            (r"/article/([0-9]+)", ArticleHandler),
-        ])
-
-    If there's no match for the current request's host, then ``default_host``
-    parameter value is matched against host regular expressions.
-
-
-    .. warning::
-
-       Applications that do not use TLS may be vulnerable to :ref:`DNS
-       rebinding ` attacks. This attack is especially
-       relevant to applications that only listen on ``127.0.0.1` or
-       other private networks. Appropriate host patterns must be used
-       (instead of the default of ``r'.*'``) to prevent this risk. The
-       ``default_host`` argument must not be used in applications that
-       may be vulnerable to DNS rebinding.
-
-    You can serve static files by sending the ``static_path`` setting
-    as a keyword argument. We will serve those files from the
-    ``/static/`` URI (this is configurable with the
-    ``static_url_prefix`` setting), and we will serve ``/favicon.ico``
-    and ``/robots.txt`` from the same directory.  A custom subclass of
-    `StaticFileHandler` can be specified with the
-    ``static_handler_class`` setting.
-
-    .. versionchanged:: 4.5
-       Integration with the new `tornado.routing` module.
-
-    """
-    def __init__(self, handlers=None, default_host=None, transforms=None,
-                 **settings):
-        if transforms is None:
-            self.transforms = []
-            if settings.get("compress_response") or settings.get("gzip"):
-                self.transforms.append(GZipContentEncoding)
-        else:
-            self.transforms = transforms
-        self.default_host = default_host
-        self.settings = settings
-        self.ui_modules = {'linkify': _linkify,
-                           'xsrf_form_html': _xsrf_form_html,
-                           'Template': TemplateModule,
-                           }
-        self.ui_methods = {}
-        self._load_ui_modules(settings.get("ui_modules", {}))
-        self._load_ui_methods(settings.get("ui_methods", {}))
-        if self.settings.get("static_path"):
-            path = self.settings["static_path"]
-            handlers = list(handlers or [])
-            static_url_prefix = settings.get("static_url_prefix",
-                                             "/static/")
-            static_handler_class = settings.get("static_handler_class",
-                                                StaticFileHandler)
-            static_handler_args = settings.get("static_handler_args", {})
-            static_handler_args['path'] = path
-            for pattern in [re.escape(static_url_prefix) + r"(.*)",
-                            r"/(favicon\.ico)", r"/(robots\.txt)"]:
-                handlers.insert(0, (pattern, static_handler_class,
-                                    static_handler_args))
-
-        if self.settings.get('debug'):
-            self.settings.setdefault('autoreload', True)
-            self.settings.setdefault('compiled_template_cache', False)
-            self.settings.setdefault('static_hash_cache', False)
-            self.settings.setdefault('serve_traceback', True)
-
-        self.wildcard_router = _ApplicationRouter(self, handlers)
-        self.default_router = _ApplicationRouter(self, [
-            Rule(AnyMatches(), self.wildcard_router)
-        ])
-
-        # Automatically reload modified modules
-        if self.settings.get('autoreload'):
-            from tornado import autoreload
-            autoreload.start()
-
-    def listen(self, port, address="", **kwargs):
-        """Starts an HTTP server for this application on the given port.
-
-        This is a convenience alias for creating an `.HTTPServer`
-        object and calling its listen method.  Keyword arguments not
-        supported by `HTTPServer.listen <.TCPServer.listen>` are passed to the
-        `.HTTPServer` constructor.  For advanced uses
-        (e.g. multi-process mode), do not use this method; create an
-        `.HTTPServer` and call its
-        `.TCPServer.bind`/`.TCPServer.start` methods directly.
-
-        Note that after calling this method you still need to call
-        ``IOLoop.current().start()`` to start the server.
-
-        Returns the `.HTTPServer` object.
-
-        .. versionchanged:: 4.3
-           Now returns the `.HTTPServer` object.
-        """
-        # import is here rather than top level because HTTPServer
-        # is not importable on appengine
-        from tornado.httpserver import HTTPServer
-        server = HTTPServer(self, **kwargs)
-        server.listen(port, address)
-        return server
-
-    def add_handlers(self, host_pattern, host_handlers):
-        """Appends the given handlers to our handler list.
-
-        Host patterns are processed sequentially in the order they were
-        added. All matching patterns will be considered.
-        """
-        host_matcher = HostMatches(host_pattern)
-        rule = Rule(host_matcher, _ApplicationRouter(self, host_handlers))
-
-        self.default_router.rules.insert(-1, rule)
-
-        if self.default_host is not None:
-            self.wildcard_router.add_rules([(
-                DefaultHostMatches(self, host_matcher.host_pattern),
-                host_handlers
-            )])
-
-    def add_transform(self, transform_class):
-        self.transforms.append(transform_class)
-
-    def _load_ui_methods(self, methods):
-        if isinstance(methods, types.ModuleType):
-            self._load_ui_methods(dict((n, getattr(methods, n))
-                                       for n in dir(methods)))
-        elif isinstance(methods, list):
-            for m in methods:
-                self._load_ui_methods(m)
-        else:
-            for name, fn in methods.items():
-                if not name.startswith("_") and hasattr(fn, "__call__") \
-                        and name[0].lower() == name[0]:
-                    self.ui_methods[name] = fn
-
-    def _load_ui_modules(self, modules):
-        if isinstance(modules, types.ModuleType):
-            self._load_ui_modules(dict((n, getattr(modules, n))
-                                       for n in dir(modules)))
-        elif isinstance(modules, list):
-            for m in modules:
-                self._load_ui_modules(m)
-        else:
-            assert isinstance(modules, dict)
-            for name, cls in modules.items():
-                try:
-                    if issubclass(cls, UIModule):
-                        self.ui_modules[name] = cls
-                except TypeError:
-                    pass
-
-    def __call__(self, request):
-        # Legacy HTTPServer interface
-        dispatcher = self.find_handler(request)
-        return dispatcher.execute()
-
-    def find_handler(self, request, **kwargs):
-        route = self.default_router.find_handler(request)
-        if route is not None:
-            return route
-
-        if self.settings.get('default_handler_class'):
-            return self.get_handler_delegate(
-                request,
-                self.settings['default_handler_class'],
-                self.settings.get('default_handler_args', {}))
-
-        return self.get_handler_delegate(
-            request, ErrorHandler, {'status_code': 404})
-
-    def get_handler_delegate(self, request, target_class, target_kwargs=None,
-                             path_args=None, path_kwargs=None):
-        """Returns `~.httputil.HTTPMessageDelegate` that can serve a request
-        for application and `RequestHandler` subclass.
-
-        :arg httputil.HTTPServerRequest request: current HTTP request.
-        :arg RequestHandler target_class: a `RequestHandler` class.
-        :arg dict target_kwargs: keyword arguments for ``target_class`` constructor.
-        :arg list path_args: positional arguments for ``target_class`` HTTP method that
-            will be executed while handling a request (``get``, ``post`` or any other).
-        :arg dict path_kwargs: keyword arguments for ``target_class`` HTTP method.
-        """
-        return _HandlerDelegate(
-            self, request, target_class, target_kwargs, path_args, path_kwargs)
-
-    def reverse_url(self, name, *args):
-        """Returns a URL path for handler named ``name``
-
-        The handler must be added to the application as a named `URLSpec`.
-
-        Args will be substituted for capturing groups in the `URLSpec` regex.
-        They will be converted to strings if necessary, encoded as utf8,
-        and url-escaped.
-        """
-        reversed_url = self.default_router.reverse_url(name, *args)
-        if reversed_url is not None:
-            return reversed_url
-
-        raise KeyError("%s not found in named urls" % name)
-
-    def log_request(self, handler):
-        """Writes a completed HTTP request to the logs.
-
-        By default writes to the python root logger.  To change
-        this behavior either subclass Application and override this method,
-        or pass a function in the application settings dictionary as
-        ``log_function``.
-        """
-        if "log_function" in self.settings:
-            self.settings["log_function"](handler)
-            return
-        if handler.get_status() < 400:
-            log_method = access_log.info
-        elif handler.get_status() < 500:
-            log_method = access_log.warning
-        else:
-            log_method = access_log.error
-        request_time = 1000.0 * handler.request.request_time()
-        log_method("%d %s %.2fms", handler.get_status(),
-                   handler._request_summary(), request_time)
-
-
-class _HandlerDelegate(httputil.HTTPMessageDelegate):
-    def __init__(self, application, request, handler_class, handler_kwargs,
-                 path_args, path_kwargs):
-        self.application = application
-        self.connection = request.connection
-        self.request = request
-        self.handler_class = handler_class
-        self.handler_kwargs = handler_kwargs or {}
-        self.path_args = path_args or []
-        self.path_kwargs = path_kwargs or {}
-        self.chunks = []
-        self.stream_request_body = _has_stream_request_body(self.handler_class)
-
-    def headers_received(self, start_line, headers):
-        if self.stream_request_body:
-            self.request.body = Future()
-            return self.execute()
-
-    def data_received(self, data):
-        if self.stream_request_body:
-            return self.handler.data_received(data)
-        else:
-            self.chunks.append(data)
-
-    def finish(self):
-        if self.stream_request_body:
-            future_set_result_unless_cancelled(self.request.body, None)
-        else:
-            self.request.body = b''.join(self.chunks)
-            self.request._parse_body()
-            self.execute()
-
-    def on_connection_close(self):
-        if self.stream_request_body:
-            self.handler.on_connection_close()
-        else:
-            self.chunks = None
-
-    def execute(self):
-        # If template cache is disabled (usually in the debug mode),
-        # re-compile templates and reload static files on every
-        # request so you don't need to restart to see changes
-        if not self.application.settings.get("compiled_template_cache", True):
-            with RequestHandler._template_loader_lock:
-                for loader in RequestHandler._template_loaders.values():
-                    loader.reset()
-        if not self.application.settings.get('static_hash_cache', True):
-            StaticFileHandler.reset()
-
-        self.handler = self.handler_class(self.application, self.request,
-                                          **self.handler_kwargs)
-        transforms = [t(self.request) for t in self.application.transforms]
-
-        if self.stream_request_body:
-            self.handler._prepared_future = Future()
-        # Note that if an exception escapes handler._execute it will be
-        # trapped in the Future it returns (which we are ignoring here,
-        # leaving it to be logged when the Future is GC'd).
-        # However, that shouldn't happen because _execute has a blanket
-        # except handler, and we cannot easily access the IOLoop here to
-        # call add_future (because of the requirement to remain compatible
-        # with WSGI)
-        self.handler._execute(transforms, *self.path_args,
-                              **self.path_kwargs)
-        # If we are streaming the request body, then execute() is finished
-        # when the handler has prepared to receive the body.  If not,
-        # it doesn't matter when execute() finishes (so we return None)
-        return self.handler._prepared_future
-
-
-class HTTPError(Exception):
-    """An exception that will turn into an HTTP error response.
-
-    Raising an `HTTPError` is a convenient alternative to calling
-    `RequestHandler.send_error` since it automatically ends the
-    current function.
-
-    To customize the response sent with an `HTTPError`, override
-    `RequestHandler.write_error`.
-
-    :arg int status_code: HTTP status code.  Must be listed in
-        `httplib.responses ` unless the ``reason``
-        keyword argument is given.
-    :arg str log_message: Message to be written to the log for this error
-        (will not be shown to the user unless the `Application` is in debug
-        mode).  May contain ``%s``-style placeholders, which will be filled
-        in with remaining positional parameters.
-    :arg str reason: Keyword-only argument.  The HTTP "reason" phrase
-        to pass in the status line along with ``status_code``.  Normally
-        determined automatically from ``status_code``, but can be used
-        to use a non-standard numeric code.
-    """
-    def __init__(self, status_code=500, log_message=None, *args, **kwargs):
-        self.status_code = status_code
-        self.log_message = log_message
-        self.args = args
-        self.reason = kwargs.get('reason', None)
-        if log_message and not args:
-            self.log_message = log_message.replace('%', '%%')
-
-    def __str__(self):
-        message = "HTTP %d: %s" % (
-            self.status_code,
-            self.reason or httputil.responses.get(self.status_code, 'Unknown'))
-        if self.log_message:
-            return message + " (" + (self.log_message % self.args) + ")"
-        else:
-            return message
-
-
-class Finish(Exception):
-    """An exception that ends the request without producing an error response.
-
-    When `Finish` is raised in a `RequestHandler`, the request will
-    end (calling `RequestHandler.finish` if it hasn't already been
-    called), but the error-handling methods (including
-    `RequestHandler.write_error`) will not be called.
-
-    If `Finish()` was created with no arguments, the pending response
-    will be sent as-is. If `Finish()` was given an argument, that
-    argument will be passed to `RequestHandler.finish()`.
-
-    This can be a more convenient way to implement custom error pages
-    than overriding ``write_error`` (especially in library code)::
-
-        if self.current_user is None:
-            self.set_status(401)
-            self.set_header('WWW-Authenticate', 'Basic realm="something"')
-            raise Finish()
-
-    .. versionchanged:: 4.3
-       Arguments passed to ``Finish()`` will be passed on to
-       `RequestHandler.finish`.
-    """
-    pass
-
-
-class MissingArgumentError(HTTPError):
-    """Exception raised by `RequestHandler.get_argument`.
-
-    This is a subclass of `HTTPError`, so if it is uncaught a 400 response
-    code will be used instead of 500 (and a stack trace will not be logged).
-
-    .. versionadded:: 3.1
-    """
-    def __init__(self, arg_name):
-        super(MissingArgumentError, self).__init__(
-            400, 'Missing argument %s' % arg_name)
-        self.arg_name = arg_name
-
-
-class ErrorHandler(RequestHandler):
-    """Generates an error response with ``status_code`` for all requests."""
-    def initialize(self, status_code):
-        self.set_status(status_code)
-
-    def prepare(self):
-        raise HTTPError(self._status_code)
-
-    def check_xsrf_cookie(self):
-        # POSTs to an ErrorHandler don't actually have side effects,
-        # so we don't need to check the xsrf token.  This allows POSTs
-        # to the wrong url to return a 404 instead of 403.
-        pass
-
-
-class RedirectHandler(RequestHandler):
-    """Redirects the client to the given URL for all GET requests.
-
-    You should provide the keyword argument ``url`` to the handler, e.g.::
-
-        application = web.Application([
-            (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}),
-        ])
-
-    `RedirectHandler` supports regular expression substitutions. E.g., to
-    swap the first and second parts of a path while preserving the remainder::
-
-        application = web.Application([
-            (r"/(.*?)/(.*?)/(.*)", web.RedirectHandler, {"url": "/{1}/{0}/{2}"}),
-        ])
-
-    The final URL is formatted with `str.format` and the substrings that match
-    the capturing groups. In the above example, a request to "/a/b/c" would be
-    formatted like::
-
-        str.format("/{1}/{0}/{2}", "a", "b", "c")  # -> "/b/a/c"
-
-    Use Python's :ref:`format string syntax ` to customize how
-    values are substituted.
-
-    .. versionchanged:: 4.5
-       Added support for substitutions into the destination URL.
-
-    .. versionchanged:: 5.0
-       If any query arguments are present, they will be copied to the
-       destination URL.
-    """
-    def initialize(self, url, permanent=True):
-        self._url = url
-        self._permanent = permanent
-
-    def get(self, *args):
-        to_url = self._url.format(*args)
-        if self.request.query_arguments:
-            to_url = httputil.url_concat(
-                to_url, list(httputil.qs_to_qsl(self.request.query_arguments)))
-        self.redirect(to_url, permanent=self._permanent)
-
-
-class StaticFileHandler(RequestHandler):
-    """A simple handler that can serve static content from a directory.
-
-    A `StaticFileHandler` is configured automatically if you pass the
-    ``static_path`` keyword argument to `Application`.  This handler
-    can be customized with the ``static_url_prefix``, ``static_handler_class``,
-    and ``static_handler_args`` settings.
-
-    To map an additional path to this handler for a static data directory
-    you would add a line to your application like::
-
-        application = web.Application([
-            (r"/content/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
-        ])
-
-    The handler constructor requires a ``path`` argument, which specifies the
-    local root directory of the content to be served.
-
-    Note that a capture group in the regex is required to parse the value for
-    the ``path`` argument to the get() method (different than the constructor
-    argument above); see `URLSpec` for details.
-
-    To serve a file like ``index.html`` automatically when a directory is
-    requested, set ``static_handler_args=dict(default_filename="index.html")``
-    in your application settings, or add ``default_filename`` as an initializer
-    argument for your ``StaticFileHandler``.
-
-    To maximize the effectiveness of browser caching, this class supports
-    versioned urls (by default using the argument ``?v=``).  If a version
-    is given, we instruct the browser to cache this file indefinitely.
-    `make_static_url` (also available as `RequestHandler.static_url`) can
-    be used to construct a versioned url.
-
-    This handler is intended primarily for use in development and light-duty
-    file serving; for heavy traffic it will be more efficient to use
-    a dedicated static file server (such as nginx or Apache).  We support
-    the HTTP ``Accept-Ranges`` mechanism to return partial content (because
-    some browsers require this functionality to be present to seek in
-    HTML5 audio or video).
-
-    **Subclassing notes**
-
-    This class is designed to be extensible by subclassing, but because
-    of the way static urls are generated with class methods rather than
-    instance methods, the inheritance patterns are somewhat unusual.
-    Be sure to use the ``@classmethod`` decorator when overriding a
-    class method.  Instance methods may use the attributes ``self.path``
-    ``self.absolute_path``, and ``self.modified``.
-
-    Subclasses should only override methods discussed in this section;
-    overriding other methods is error-prone.  Overriding
-    ``StaticFileHandler.get`` is particularly problematic due to the
-    tight coupling with ``compute_etag`` and other methods.
-
-    To change the way static urls are generated (e.g. to match the behavior
-    of another server or CDN), override `make_static_url`, `parse_url_path`,
-    `get_cache_time`, and/or `get_version`.
-
-    To replace all interaction with the filesystem (e.g. to serve
-    static content from a database), override `get_content`,
-    `get_content_size`, `get_modified_time`, `get_absolute_path`, and
-    `validate_absolute_path`.
-
-    .. versionchanged:: 3.1
-       Many of the methods for subclasses were added in Tornado 3.1.
-    """
-    CACHE_MAX_AGE = 86400 * 365 * 10  # 10 years
-
-    _static_hashes = {}  # type: typing.Dict
-    _lock = threading.Lock()  # protects _static_hashes
-
-    def initialize(self, path, default_filename=None):
-        self.root = path
-        self.default_filename = default_filename
-
-    @classmethod
-    def reset(cls):
-        with cls._lock:
-            cls._static_hashes = {}
-
-    def head(self, path):
-        return self.get(path, include_body=False)
-
-    @gen.coroutine
-    def get(self, path, include_body=True):
-        # Set up our path instance variables.
-        self.path = self.parse_url_path(path)
-        del path  # make sure we don't refer to path instead of self.path again
-        absolute_path = self.get_absolute_path(self.root, self.path)
-        self.absolute_path = self.validate_absolute_path(
-            self.root, absolute_path)
-        if self.absolute_path is None:
-            return
-
-        self.modified = self.get_modified_time()
-        self.set_headers()
-
-        if self.should_return_304():
-            self.set_status(304)
-            return
-
-        request_range = None
-        range_header = self.request.headers.get("Range")
-        if range_header:
-            # As per RFC 2616 14.16, if an invalid Range header is specified,
-            # the request will be treated as if the header didn't exist.
-            request_range = httputil._parse_request_range(range_header)
-
-        size = self.get_content_size()
-        if request_range:
-            start, end = request_range
-            if (start is not None and start >= size) or end == 0:
-                # As per RFC 2616 14.35.1, a range is not satisfiable only: if
-                # the first requested byte is equal to or greater than the
-                # content, or when a suffix with length 0 is specified
-                self.set_status(416)  # Range Not Satisfiable
-                self.set_header("Content-Type", "text/plain")
-                self.set_header("Content-Range", "bytes */%s" % (size, ))
-                return
-            if start is not None and start < 0:
-                start += size
-            if end is not None and end > size:
-                # Clients sometimes blindly use a large range to limit their
-                # download size; cap the endpoint at the actual file size.
-                end = size
-            # Note: only return HTTP 206 if less than the entire range has been
-            # requested. Not only is this semantically correct, but Chrome
-            # refuses to play audio if it gets an HTTP 206 in response to
-            # ``Range: bytes=0-``.
-            if size != (end or size) - (start or 0):
-                self.set_status(206)  # Partial Content
-                self.set_header("Content-Range",
-                                httputil._get_content_range(start, end, size))
-        else:
-            start = end = None
-
-        if start is not None and end is not None:
-            content_length = end - start
-        elif end is not None:
-            content_length = end
-        elif start is not None:
-            content_length = size - start
-        else:
-            content_length = size
-        self.set_header("Content-Length", content_length)
-
-        if include_body:
-            content = self.get_content(self.absolute_path, start, end)
-            if isinstance(content, bytes):
-                content = [content]
-            for chunk in content:
-                try:
-                    self.write(chunk)
-                    yield self.flush()
-                except iostream.StreamClosedError:
-                    return
-        else:
-            assert self.request.method == "HEAD"
-
-    def compute_etag(self):
-        """Sets the ``Etag`` header based on static url version.
-
-        This allows efficient ``If-None-Match`` checks against cached
-        versions, and sends the correct ``Etag`` for a partial response
-        (i.e. the same ``Etag`` as the full file).
-
-        .. versionadded:: 3.1
-        """
-        version_hash = self._get_cached_version(self.absolute_path)
-        if not version_hash:
-            return None
-        return '"%s"' % (version_hash, )
-
-    def set_headers(self):
-        """Sets the content and caching headers on the response.
-
-        .. versionadded:: 3.1
-        """
-        self.set_header("Accept-Ranges", "bytes")
-        self.set_etag_header()
-
-        if self.modified is not None:
-            self.set_header("Last-Modified", self.modified)
-
-        content_type = self.get_content_type()
-        if content_type:
-            self.set_header("Content-Type", content_type)
-
-        cache_time = self.get_cache_time(self.path, self.modified,
-                                         content_type)
-        if cache_time > 0:
-            self.set_header("Expires", datetime.datetime.utcnow() +
-                            datetime.timedelta(seconds=cache_time))
-            self.set_header("Cache-Control", "max-age=" + str(cache_time))
-
-        self.set_extra_headers(self.path)
-
-    def should_return_304(self):
-        """Returns True if the headers indicate that we should return 304.
-
-        .. versionadded:: 3.1
-        """
-        # If client sent If-None-Match, use it, ignore If-Modified-Since
-        if self.request.headers.get('If-None-Match'):
-            return self.check_etag_header()
-
-        # Check the If-Modified-Since, and don't send the result if the
-        # content has not been modified
-        ims_value = self.request.headers.get("If-Modified-Since")
-        if ims_value is not None:
-            date_tuple = email.utils.parsedate(ims_value)
-            if date_tuple is not None:
-                if_since = datetime.datetime(*date_tuple[:6])
-                if if_since >= self.modified:
-                    return True
-
-        return False
-
-    @classmethod
-    def get_absolute_path(cls, root, path):
-        """Returns the absolute location of ``path`` relative to ``root``.
-
-        ``root`` is the path configured for this `StaticFileHandler`
-        (in most cases the ``static_path`` `Application` setting).
-
-        This class method may be overridden in subclasses.  By default
-        it returns a filesystem path, but other strings may be used
-        as long as they are unique and understood by the subclass's
-        overridden `get_content`.
-
-        .. versionadded:: 3.1
-        """
-        abspath = os.path.abspath(os.path.join(root, path))
-        return abspath
-
-    def validate_absolute_path(self, root, absolute_path):
-        """Validate and return the absolute path.
-
-        ``root`` is the configured path for the `StaticFileHandler`,
-        and ``path`` is the result of `get_absolute_path`
-
-        This is an instance method called during request processing,
-        so it may raise `HTTPError` or use methods like
-        `RequestHandler.redirect` (return None after redirecting to
-        halt further processing).  This is where 404 errors for missing files
-        are generated.
-
-        This method may modify the path before returning it, but note that
-        any such modifications will not be understood by `make_static_url`.
-
-        In instance methods, this method's result is available as
-        ``self.absolute_path``.
-
-        .. versionadded:: 3.1
-        """
-        # os.path.abspath strips a trailing /.
-        # We must add it back to `root` so that we only match files
-        # in a directory named `root` instead of files starting with
-        # that prefix.
-        root = os.path.abspath(root)
-        if not root.endswith(os.path.sep):
-            # abspath always removes a trailing slash, except when
-            # root is '/'. This is an unusual case, but several projects
-            # have independently discovered this technique to disable
-            # Tornado's path validation and (hopefully) do their own,
-            # so we need to support it.
-            root += os.path.sep
-        # The trailing slash also needs to be temporarily added back
-        # the requested path so a request to root/ will match.
-        if not (absolute_path + os.path.sep).startswith(root):
-            raise HTTPError(403, "%s is not in root static directory",
-                            self.path)
-        if (os.path.isdir(absolute_path) and
-                self.default_filename is not None):
-            # need to look at the request.path here for when path is empty
-            # but there is some prefix to the path that was already
-            # trimmed by the routing
-            if not self.request.path.endswith("/"):
-                self.redirect(self.request.path + "/", permanent=True)
-                return
-            absolute_path = os.path.join(absolute_path, self.default_filename)
-        if not os.path.exists(absolute_path):
-            raise HTTPError(404)
-        if not os.path.isfile(absolute_path):
-            raise HTTPError(403, "%s is not a file", self.path)
-        return absolute_path
-
-    @classmethod
-    def get_content(cls, abspath, start=None, end=None):
-        """Retrieve the content of the requested resource which is located
-        at the given absolute path.
-
-        This class method may be overridden by subclasses.  Note that its
-        signature is different from other overridable class methods
-        (no ``settings`` argument); this is deliberate to ensure that
-        ``abspath`` is able to stand on its own as a cache key.
-
-        This method should either return a byte string or an iterator
-        of byte strings.  The latter is preferred for large files
-        as it helps reduce memory fragmentation.
-
-        .. versionadded:: 3.1
-        """
-        with open(abspath, "rb") as file:
-            if start is not None:
-                file.seek(start)
-            if end is not None:
-                remaining = end - (start or 0)
-            else:
-                remaining = None
-            while True:
-                chunk_size = 64 * 1024
-                if remaining is not None and remaining < chunk_size:
-                    chunk_size = remaining
-                chunk = file.read(chunk_size)
-                if chunk:
-                    if remaining is not None:
-                        remaining -= len(chunk)
-                    yield chunk
-                else:
-                    if remaining is not None:
-                        assert remaining == 0
-                    return
-
-    @classmethod
-    def get_content_version(cls, abspath):
-        """Returns a version string for the resource at the given path.
-
-        This class method may be overridden by subclasses.  The
-        default implementation is a hash of the file's contents.
-
-        .. versionadded:: 3.1
-        """
-        data = cls.get_content(abspath)
-        hasher = hashlib.md5()
-        if isinstance(data, bytes):
-            hasher.update(data)
-        else:
-            for chunk in data:
-                hasher.update(chunk)
-        return hasher.hexdigest()
-
-    def _stat(self):
-        if not hasattr(self, '_stat_result'):
-            self._stat_result = os.stat(self.absolute_path)
-        return self._stat_result
-
-    def get_content_size(self):
-        """Retrieve the total size of the resource at the given path.
-
-        This method may be overridden by subclasses.
-
-        .. versionadded:: 3.1
-
-        .. versionchanged:: 4.0
-           This method is now always called, instead of only when
-           partial results are requested.
-        """
-        stat_result = self._stat()
-        return stat_result[stat.ST_SIZE]
-
-    def get_modified_time(self):
-        """Returns the time that ``self.absolute_path`` was last modified.
-
-        May be overridden in subclasses.  Should return a `~datetime.datetime`
-        object or None.
-
-        .. versionadded:: 3.1
-        """
-        stat_result = self._stat()
-        modified = datetime.datetime.utcfromtimestamp(
-            stat_result[stat.ST_MTIME])
-        return modified
-
-    def get_content_type(self):
-        """Returns the ``Content-Type`` header to be used for this request.
-
-        .. versionadded:: 3.1
-        """
-        mime_type, encoding = mimetypes.guess_type(self.absolute_path)
-        # per RFC 6713, use the appropriate type for a gzip compressed file
-        if encoding == "gzip":
-            return "application/gzip"
-        # As of 2015-07-21 there is no bzip2 encoding defined at
-        # http://www.iana.org/assignments/media-types/media-types.xhtml
-        # So for that (and any other encoding), use octet-stream.
-        elif encoding is not None:
-            return "application/octet-stream"
-        elif mime_type is not None:
-            return mime_type
-        # if mime_type not detected, use application/octet-stream
-        else:
-            return "application/octet-stream"
-
-    def set_extra_headers(self, path):
-        """For subclass to add extra headers to the response"""
-        pass
-
-    def get_cache_time(self, path, modified, mime_type):
-        """Override to customize cache control behavior.
-
-        Return a positive number of seconds to make the result
-        cacheable for that amount of time or 0 to mark resource as
-        cacheable for an unspecified amount of time (subject to
-        browser heuristics).
-
-        By default returns cache expiry of 10 years for resources requested
-        with ``v`` argument.
-        """
-        return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0
-
-    @classmethod
-    def make_static_url(cls, settings, path, include_version=True):
-        """Constructs a versioned url for the given path.
-
-        This method may be overridden in subclasses (but note that it
-        is a class method rather than an instance method).  Subclasses
-        are only required to implement the signature
-        ``make_static_url(cls, settings, path)``; other keyword
-        arguments may be passed through `~RequestHandler.static_url`
-        but are not standard.
-
-        ``settings`` is the `Application.settings` dictionary.  ``path``
-        is the static path being requested.  The url returned should be
-        relative to the current host.
-
-        ``include_version`` determines whether the generated URL should
-        include the query string containing the version hash of the
-        file corresponding to the given ``path``.
-
-        """
-        url = settings.get('static_url_prefix', '/static/') + path
-        if not include_version:
-            return url
-
-        version_hash = cls.get_version(settings, path)
-        if not version_hash:
-            return url
-
-        return '%s?v=%s' % (url, version_hash)
-
-    def parse_url_path(self, url_path):
-        """Converts a static URL path into a filesystem path.
-
-        ``url_path`` is the path component of the URL with
-        ``static_url_prefix`` removed.  The return value should be
-        filesystem path relative to ``static_path``.
-
-        This is the inverse of `make_static_url`.
-        """
-        if os.path.sep != "/":
-            url_path = url_path.replace("/", os.path.sep)
-        return url_path
-
-    @classmethod
-    def get_version(cls, settings, path):
-        """Generate the version string to be used in static URLs.
-
-        ``settings`` is the `Application.settings` dictionary and ``path``
-        is the relative location of the requested asset on the filesystem.
-        The returned value should be a string, or ``None`` if no version
-        could be determined.
-
-        .. versionchanged:: 3.1
-           This method was previously recommended for subclasses to override;
-           `get_content_version` is now preferred as it allows the base
-           class to handle caching of the result.
-        """
-        abs_path = cls.get_absolute_path(settings['static_path'], path)
-        return cls._get_cached_version(abs_path)
-
-    @classmethod
-    def _get_cached_version(cls, abs_path):
-        with cls._lock:
-            hashes = cls._static_hashes
-            if abs_path not in hashes:
-                try:
-                    hashes[abs_path] = cls.get_content_version(abs_path)
-                except Exception:
-                    gen_log.error("Could not open static file %r", abs_path)
-                    hashes[abs_path] = None
-            hsh = hashes.get(abs_path)
-            if hsh:
-                return hsh
-        return None
-
-
-class FallbackHandler(RequestHandler):
-    """A `RequestHandler` that wraps another HTTP server callback.
-
-    The fallback is a callable object that accepts an
-    `~.httputil.HTTPServerRequest`, such as an `Application` or
-    `tornado.wsgi.WSGIContainer`.  This is most useful to use both
-    Tornado ``RequestHandlers`` and WSGI in the same server.  Typical
-    usage::
-
-        wsgi_app = tornado.wsgi.WSGIContainer(
-            django.core.handlers.wsgi.WSGIHandler())
-        application = tornado.web.Application([
-            (r"/foo", FooHandler),
-            (r".*", FallbackHandler, dict(fallback=wsgi_app),
-        ])
-    """
-    def initialize(self, fallback):
-        self.fallback = fallback
-
-    def prepare(self):
-        self.fallback(self.request)
-        self._finished = True
-        self.on_finish()
-
-
-class OutputTransform(object):
-    """A transform modifies the result of an HTTP request (e.g., GZip encoding)
-
-    Applications are not expected to create their own OutputTransforms
-    or interact with them directly; the framework chooses which transforms
-    (if any) to apply.
-    """
-    def __init__(self, request):
-        pass
-
-    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        # type: (int, httputil.HTTPHeaders, bytes, bool) -> typing.Tuple[int, httputil.HTTPHeaders, bytes] # noqa: E501
-        return status_code, headers, chunk
-
-    def transform_chunk(self, chunk, finishing):
-        return chunk
-
-
-class GZipContentEncoding(OutputTransform):
-    """Applies the gzip content encoding to the response.
-
-    See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
-
-    .. versionchanged:: 4.0
-        Now compresses all mime types beginning with ``text/``, instead
-        of just a whitelist. (the whitelist is still used for certain
-        non-text mime types).
-    """
-    # Whitelist of compressible mime types (in addition to any types
-    # beginning with "text/").
-    CONTENT_TYPES = set(["application/javascript", "application/x-javascript",
-                         "application/xml", "application/atom+xml",
-                         "application/json", "application/xhtml+xml",
-                         "image/svg+xml"])
-    # Python's GzipFile defaults to level 9, while most other gzip
-    # tools (including gzip itself) default to 6, which is probably a
-    # better CPU/size tradeoff.
-    GZIP_LEVEL = 6
-    # Responses that are too short are unlikely to benefit from gzipping
-    # after considering the "Content-Encoding: gzip" header and the header
-    # inside the gzip encoding.
-    # Note that responses written in multiple chunks will be compressed
-    # regardless of size.
-    MIN_LENGTH = 1024
-
-    def __init__(self, request):
-        self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "")
-
-    def _compressible_type(self, ctype):
-        return ctype.startswith('text/') or ctype in self.CONTENT_TYPES
-
-    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        # type: (int, httputil.HTTPHeaders, bytes, bool) -> typing.Tuple[int, httputil.HTTPHeaders, bytes] # noqa: E501
-        # TODO: can/should this type be inherited from the superclass?
-        if 'Vary' in headers:
-            headers['Vary'] += ', Accept-Encoding'
-        else:
-            headers['Vary'] = 'Accept-Encoding'
-        if self._gzipping:
-            ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
-            self._gzipping = self._compressible_type(ctype) and \
-                (not finishing or len(chunk) >= self.MIN_LENGTH) and \
-                ("Content-Encoding" not in headers)
-        if self._gzipping:
-            headers["Content-Encoding"] = "gzip"
-            self._gzip_value = BytesIO()
-            self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value,
-                                            compresslevel=self.GZIP_LEVEL)
-            chunk = self.transform_chunk(chunk, finishing)
-            if "Content-Length" in headers:
-                # The original content length is no longer correct.
-                # If this is the last (and only) chunk, we can set the new
-                # content-length; otherwise we remove it and fall back to
-                # chunked encoding.
-                if finishing:
-                    headers["Content-Length"] = str(len(chunk))
-                else:
-                    del headers["Content-Length"]
-        return status_code, headers, chunk
-
-    def transform_chunk(self, chunk, finishing):
-        if self._gzipping:
-            self._gzip_file.write(chunk)
-            if finishing:
-                self._gzip_file.close()
-            else:
-                self._gzip_file.flush()
-            chunk = self._gzip_value.getvalue()
-            self._gzip_value.truncate(0)
-            self._gzip_value.seek(0)
-        return chunk
-
-
-def authenticated(method):
-    """Decorate methods with this to require that the user be logged in.
-
-    If the user is not logged in, they will be redirected to the configured
-    `login url `.
-
-    If you configure a login url with a query parameter, Tornado will
-    assume you know what you're doing and use it as-is.  If not, it
-    will add a `next` parameter so the login page knows where to send
-    you once you're logged in.
-    """
-    @functools.wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if not self.current_user:
-            if self.request.method in ("GET", "HEAD"):
-                url = self.get_login_url()
-                if "?" not in url:
-                    if urlparse.urlsplit(url).scheme:
-                        # if login url is absolute, make next absolute too
-                        next_url = self.request.full_url()
-                    else:
-                        next_url = self.request.uri
-                    url += "?" + urlencode(dict(next=next_url))
-                self.redirect(url)
-                return
-            raise HTTPError(403)
-        return method(self, *args, **kwargs)
-    return wrapper
-
-
-class UIModule(object):
-    """A re-usable, modular UI unit on a page.
-
-    UI modules often execute additional queries, and they can include
-    additional CSS and JavaScript that will be included in the output
-    page, which is automatically inserted on page render.
-
-    Subclasses of UIModule must override the `render` method.
-    """
-    def __init__(self, handler):
-        self.handler = handler
-        self.request = handler.request
-        self.ui = handler.ui
-        self.locale = handler.locale
-
-    @property
-    def current_user(self):
-        return self.handler.current_user
-
-    def render(self, *args, **kwargs):
-        """Override in subclasses to return this module's output."""
-        raise NotImplementedError()
-
-    def embedded_javascript(self):
-        """Override to return a JavaScript string
-        to be embedded in the page."""
-        return None
-
-    def javascript_files(self):
-        """Override to return a list of JavaScript files needed by this module.
-
-        If the return values are relative paths, they will be passed to
-        `RequestHandler.static_url`; otherwise they will be used as-is.
-        """
-        return None
-
-    def embedded_css(self):
-        """Override to return a CSS string
-        that will be embedded in the page."""
-        return None
-
-    def css_files(self):
-        """Override to returns a list of CSS files required by this module.
-
-        If the return values are relative paths, they will be passed to
-        `RequestHandler.static_url`; otherwise they will be used as-is.
-        """
-        return None
-
-    def html_head(self):
-        """Override to return an HTML string that will be put in the 
-        element.
-        """
-        return None
-
-    def html_body(self):
-        """Override to return an HTML string that will be put at the end of
-        the  element.
-        """
-        return None
-
-    def render_string(self, path, **kwargs):
-        """Renders a template and returns it as a string."""
-        return self.handler.render_string(path, **kwargs)
-
-
-class _linkify(UIModule):
-    def render(self, text, **kwargs):
-        return escape.linkify(text, **kwargs)
-
-
-class _xsrf_form_html(UIModule):
-    def render(self):
-        return self.handler.xsrf_form_html()
-
-
-class TemplateModule(UIModule):
-    """UIModule that simply renders the given template.
-
-    {% module Template("foo.html") %} is similar to {% include "foo.html" %},
-    but the module version gets its own namespace (with kwargs passed to
-    Template()) instead of inheriting the outer template's namespace.
-
-    Templates rendered through this module also get access to UIModule's
-    automatic javascript/css features.  Simply call set_resources
-    inside the template and give it keyword arguments corresponding to
-    the methods on UIModule: {{ set_resources(js_files=static_url("my.js")) }}
-    Note that these resources are output once per template file, not once
-    per instantiation of the template, so they must not depend on
-    any arguments to the template.
-    """
-    def __init__(self, handler):
-        super(TemplateModule, self).__init__(handler)
-        # keep resources in both a list and a dict to preserve order
-        self._resource_list = []
-        self._resource_dict = {}
-
-    def render(self, path, **kwargs):
-        def set_resources(**kwargs):
-            if path not in self._resource_dict:
-                self._resource_list.append(kwargs)
-                self._resource_dict[path] = kwargs
-            else:
-                if self._resource_dict[path] != kwargs:
-                    raise ValueError("set_resources called with different "
-                                     "resources for the same template")
-            return ""
-        return self.render_string(path, set_resources=set_resources,
-                                  **kwargs)
-
-    def _get_resources(self, key):
-        return (r[key] for r in self._resource_list if key in r)
-
-    def embedded_javascript(self):
-        return "\n".join(self._get_resources("embedded_javascript"))
-
-    def javascript_files(self):
-        result = []
-        for f in self._get_resources("javascript_files"):
-            if isinstance(f, (unicode_type, bytes)):
-                result.append(f)
-            else:
-                result.extend(f)
-        return result
-
-    def embedded_css(self):
-        return "\n".join(self._get_resources("embedded_css"))
-
-    def css_files(self):
-        result = []
-        for f in self._get_resources("css_files"):
-            if isinstance(f, (unicode_type, bytes)):
-                result.append(f)
-            else:
-                result.extend(f)
-        return result
-
-    def html_head(self):
-        return "".join(self._get_resources("html_head"))
-
-    def html_body(self):
-        return "".join(self._get_resources("html_body"))
-
-
-class _UIModuleNamespace(object):
-    """Lazy namespace which creates UIModule proxies bound to a handler."""
-    def __init__(self, handler, ui_modules):
-        self.handler = handler
-        self.ui_modules = ui_modules
-
-    def __getitem__(self, key):
-        return self.handler._ui_module(key, self.ui_modules[key])
-
-    def __getattr__(self, key):
-        try:
-            return self[key]
-        except KeyError as e:
-            raise AttributeError(str(e))
-
-
-if hasattr(hmac, 'compare_digest'):  # python 3.3
-    _time_independent_equals = hmac.compare_digest
-else:
-    def _time_independent_equals(a, b):
-        if len(a) != len(b):
-            return False
-        result = 0
-        if isinstance(a[0], int):  # python3 byte strings
-            for x, y in zip(a, b):
-                result |= x ^ y
-        else:  # python2
-            for x, y in zip(a, b):
-                result |= ord(x) ^ ord(y)
-        return result == 0
-
-
-def create_signed_value(secret, name, value, version=None, clock=None,
-                        key_version=None):
-    if version is None:
-        version = DEFAULT_SIGNED_VALUE_VERSION
-    if clock is None:
-        clock = time.time
-
-    timestamp = utf8(str(int(clock())))
-    value = base64.b64encode(utf8(value))
-    if version == 1:
-        signature = _create_signature_v1(secret, name, value, timestamp)
-        value = b"|".join([value, timestamp, signature])
-        return value
-    elif version == 2:
-        # The v2 format consists of a version number and a series of
-        # length-prefixed fields "%d:%s", the last of which is a
-        # signature, all separated by pipes.  All numbers are in
-        # decimal format with no leading zeros.  The signature is an
-        # HMAC-SHA256 of the whole string up to that point, including
-        # the final pipe.
-        #
-        # The fields are:
-        # - format version (i.e. 2; no length prefix)
-        # - key version (integer, default is 0)
-        # - timestamp (integer seconds since epoch)
-        # - name (not encoded; assumed to be ~alphanumeric)
-        # - value (base64-encoded)
-        # - signature (hex-encoded; no length prefix)
-        def format_field(s):
-            return utf8("%d:" % len(s)) + utf8(s)
-        to_sign = b"|".join([
-            b"2",
-            format_field(str(key_version or 0)),
-            format_field(timestamp),
-            format_field(name),
-            format_field(value),
-            b''])
-
-        if isinstance(secret, dict):
-            assert key_version is not None, 'Key version must be set when sign key dict is used'
-            assert version >= 2, 'Version must be at least 2 for key version support'
-            secret = secret[key_version]
-
-        signature = _create_signature_v2(secret, to_sign)
-        return to_sign + signature
-    else:
-        raise ValueError("Unsupported version %d" % version)
-
-
-# A leading version number in decimal
-# with no leading zeros, followed by a pipe.
-_signed_value_version_re = re.compile(br"^([1-9][0-9]*)\|(.*)$")
-
-
-def _get_version(value):
-    # Figures out what version value is.  Version 1 did not include an
-    # explicit version field and started with arbitrary base64 data,
-    # which makes this tricky.
-    m = _signed_value_version_re.match(value)
-    if m is None:
-        version = 1
-    else:
-        try:
-            version = int(m.group(1))
-            if version > 999:
-                # Certain payloads from the version-less v1 format may
-                # be parsed as valid integers.  Due to base64 padding
-                # restrictions, this can only happen for numbers whose
-                # length is a multiple of 4, so we can treat all
-                # numbers up to 999 as versions, and for the rest we
-                # fall back to v1 format.
-                version = 1
-        except ValueError:
-            version = 1
-    return version
-
-
-def decode_signed_value(secret, name, value, max_age_days=31,
-                        clock=None, min_version=None):
-    if clock is None:
-        clock = time.time
-    if min_version is None:
-        min_version = DEFAULT_SIGNED_VALUE_MIN_VERSION
-    if min_version > 2:
-        raise ValueError("Unsupported min_version %d" % min_version)
-    if not value:
-        return None
-
-    value = utf8(value)
-    version = _get_version(value)
-
-    if version < min_version:
-        return None
-    if version == 1:
-        return _decode_signed_value_v1(secret, name, value,
-                                       max_age_days, clock)
-    elif version == 2:
-        return _decode_signed_value_v2(secret, name, value,
-                                       max_age_days, clock)
-    else:
-        return None
-
-
-def _decode_signed_value_v1(secret, name, value, max_age_days, clock):
-    parts = utf8(value).split(b"|")
-    if len(parts) != 3:
-        return None
-    signature = _create_signature_v1(secret, name, parts[0], parts[1])
-    if not _time_independent_equals(parts[2], signature):
-        gen_log.warning("Invalid cookie signature %r", value)
-        return None
-    timestamp = int(parts[1])
-    if timestamp < clock() - max_age_days * 86400:
-        gen_log.warning("Expired cookie %r", value)
-        return None
-    if timestamp > clock() + 31 * 86400:
-        # _cookie_signature does not hash a delimiter between the
-        # parts of the cookie, so an attacker could transfer trailing
-        # digits from the payload to the timestamp without altering the
-        # signature.  For backwards compatibility, sanity-check timestamp
-        # here instead of modifying _cookie_signature.
-        gen_log.warning("Cookie timestamp in future; possible tampering %r",
-                        value)
-        return None
-    if parts[1].startswith(b"0"):
-        gen_log.warning("Tampered cookie %r", value)
-        return None
-    try:
-        return base64.b64decode(parts[0])
-    except Exception:
-        return None
-
-
-def _decode_fields_v2(value):
-    def _consume_field(s):
-        length, _, rest = s.partition(b':')
-        n = int(length)
-        field_value = rest[:n]
-        # In python 3, indexing bytes returns small integers; we must
-        # use a slice to get a byte string as in python 2.
-        if rest[n:n + 1] != b'|':
-            raise ValueError("malformed v2 signed value field")
-        rest = rest[n + 1:]
-        return field_value, rest
-
-    rest = value[2:]  # remove version number
-    key_version, rest = _consume_field(rest)
-    timestamp, rest = _consume_field(rest)
-    name_field, rest = _consume_field(rest)
-    value_field, passed_sig = _consume_field(rest)
-    return int(key_version), timestamp, name_field, value_field, passed_sig
-
-
-def _decode_signed_value_v2(secret, name, value, max_age_days, clock):
-    try:
-        key_version, timestamp, name_field, value_field, passed_sig = _decode_fields_v2(value)
-    except ValueError:
-        return None
-    signed_string = value[:-len(passed_sig)]
-
-    if isinstance(secret, dict):
-        try:
-            secret = secret[key_version]
-        except KeyError:
-            return None
-
-    expected_sig = _create_signature_v2(secret, signed_string)
-    if not _time_independent_equals(passed_sig, expected_sig):
-        return None
-    if name_field != utf8(name):
-        return None
-    timestamp = int(timestamp)
-    if timestamp < clock() - max_age_days * 86400:
-        # The signature has expired.
-        return None
-    try:
-        return base64.b64decode(value_field)
-    except Exception:
-        return None
-
-
-def get_signature_key_version(value):
-    value = utf8(value)
-    version = _get_version(value)
-    if version < 2:
-        return None
-    try:
-        key_version, _, _, _, _ = _decode_fields_v2(value)
-    except ValueError:
-        return None
-
-    return key_version
-
-
-def _create_signature_v1(secret, *parts):
-    hash = hmac.new(utf8(secret), digestmod=hashlib.sha1)
-    for part in parts:
-        hash.update(utf8(part))
-    return utf8(hash.hexdigest())
-
-
-def _create_signature_v2(secret, s):
-    hash = hmac.new(utf8(secret), digestmod=hashlib.sha256)
-    hash.update(utf8(s))
-    return utf8(hash.hexdigest())
-
-
-def is_absolute(path):
-    return any(path.startswith(x) for x in ["/", "http:", "https:"])
diff --git a/lib/tornado/websocket.py b/lib/tornado/websocket.py
deleted file mode 100755
index 7b77850a..00000000
--- a/lib/tornado/websocket.py
+++ /dev/null
@@ -1,1346 +0,0 @@
-"""Implementation of the WebSocket protocol.
-
-`WebSockets `_ allow for bidirectional
-communication between the browser and server.
-
-WebSockets are supported in the current versions of all major browsers,
-although older versions that do not support WebSockets are still in use
-(refer to http://caniuse.com/websockets for details).
-
-This module implements the final version of the WebSocket protocol as
-defined in `RFC 6455 `_.  Certain
-browser versions (notably Safari 5.x) implemented an earlier draft of
-the protocol (known as "draft 76") and are not compatible with this module.
-
-.. versionchanged:: 4.0
-   Removed support for the draft 76 protocol version.
-"""
-
-from __future__ import absolute_import, division, print_function
-
-import base64
-import hashlib
-import os
-import sys
-import struct
-import tornado.escape
-import tornado.web
-import zlib
-
-from tornado.concurrent import Future, future_set_result_unless_cancelled
-from tornado.escape import utf8, native_str, to_unicode
-from tornado import gen, httpclient, httputil
-from tornado.ioloop import IOLoop, PeriodicCallback
-from tornado.iostream import StreamClosedError
-from tornado.log import gen_log
-from tornado import simple_httpclient
-from tornado.queues import Queue
-from tornado.tcpclient import TCPClient
-from tornado.util import _websocket_mask, PY3
-
-if PY3:
-    from urllib.parse import urlparse  # py2
-    xrange = range
-else:
-    from urlparse import urlparse  # py3
-
-_default_max_message_size = 10 * 1024 * 1024
-
-
-class WebSocketError(Exception):
-    pass
-
-
-class WebSocketClosedError(WebSocketError):
-    """Raised by operations on a closed connection.
-
-    .. versionadded:: 3.2
-    """
-    pass
-
-
-class _DecompressTooLargeError(Exception):
-    pass
-
-
-class WebSocketHandler(tornado.web.RequestHandler):
-    """Subclass this class to create a basic WebSocket handler.
-
-    Override `on_message` to handle incoming messages, and use
-    `write_message` to send messages to the client. You can also
-    override `open` and `on_close` to handle opened and closed
-    connections.
-
-    Custom upgrade response headers can be sent by overriding
-    `~tornado.web.RequestHandler.set_default_headers` or
-    `~tornado.web.RequestHandler.prepare`.
-
-    See http://dev.w3.org/html5/websockets/ for details on the
-    JavaScript interface.  The protocol is specified at
-    http://tools.ietf.org/html/rfc6455.
-
-    Here is an example WebSocket handler that echos back all received messages
-    back to the client:
-
-    .. testcode::
-
-      class EchoWebSocket(tornado.websocket.WebSocketHandler):
-          def open(self):
-              print("WebSocket opened")
-
-          def on_message(self, message):
-              self.write_message(u"You said: " + message)
-
-          def on_close(self):
-              print("WebSocket closed")
-
-    .. testoutput::
-       :hide:
-
-    WebSockets are not standard HTTP connections. The "handshake" is
-    HTTP, but after the handshake, the protocol is
-    message-based. Consequently, most of the Tornado HTTP facilities
-    are not available in handlers of this type. The only communication
-    methods available to you are `write_message()`, `ping()`, and
-    `close()`. Likewise, your request handler class should implement
-    `open()` method rather than ``get()`` or ``post()``.
-
-    If you map the handler above to ``/websocket`` in your application, you can
-    invoke it in JavaScript with::
-
-      var ws = new WebSocket("ws://localhost:8888/websocket");
-      ws.onopen = function() {
-         ws.send("Hello, world");
-      };
-      ws.onmessage = function (evt) {
-         alert(evt.data);
-      };
-
-    This script pops up an alert box that says "You said: Hello, world".
-
-    Web browsers allow any site to open a websocket connection to any other,
-    instead of using the same-origin policy that governs other network
-    access from javascript.  This can be surprising and is a potential
-    security hole, so since Tornado 4.0 `WebSocketHandler` requires
-    applications that wish to receive cross-origin websockets to opt in
-    by overriding the `~WebSocketHandler.check_origin` method (see that
-    method's docs for details).  Failure to do so is the most likely
-    cause of 403 errors when making a websocket connection.
-
-    When using a secure websocket connection (``wss://``) with a self-signed
-    certificate, the connection from a browser may fail because it wants
-    to show the "accept this certificate" dialog but has nowhere to show it.
-    You must first visit a regular HTML page using the same certificate
-    to accept it before the websocket connection will succeed.
-
-    If the application setting ``websocket_ping_interval`` has a non-zero
-    value, a ping will be sent periodically, and the connection will be
-    closed if a response is not received before the ``websocket_ping_timeout``.
-
-    Messages larger than the ``websocket_max_message_size`` application setting
-    (default 10MiB) will not be accepted.
-
-    .. versionchanged:: 4.5
-       Added ``websocket_ping_interval``, ``websocket_ping_timeout``, and
-       ``websocket_max_message_size``.
-    """
-    def __init__(self, application, request, **kwargs):
-        super(WebSocketHandler, self).__init__(application, request, **kwargs)
-        self.ws_connection = None
-        self.close_code = None
-        self.close_reason = None
-        self.stream = None
-        self._on_close_called = False
-
-    def get(self, *args, **kwargs):
-        self.open_args = args
-        self.open_kwargs = kwargs
-
-        # Upgrade header should be present and should be equal to WebSocket
-        if self.request.headers.get("Upgrade", "").lower() != 'websocket':
-            self.set_status(400)
-            log_msg = "Can \"Upgrade\" only to \"WebSocket\"."
-            self.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        # Connection header should be upgrade.
-        # Some proxy servers/load balancers
-        # might mess with it.
-        headers = self.request.headers
-        connection = map(lambda s: s.strip().lower(),
-                         headers.get("Connection", "").split(","))
-        if 'upgrade' not in connection:
-            self.set_status(400)
-            log_msg = "\"Connection\" must be \"Upgrade\"."
-            self.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        # Handle WebSocket Origin naming convention differences
-        # The difference between version 8 and 13 is that in 8 the
-        # client sends a "Sec-Websocket-Origin" header and in 13 it's
-        # simply "Origin".
-        if "Origin" in self.request.headers:
-            origin = self.request.headers.get("Origin")
-        else:
-            origin = self.request.headers.get("Sec-Websocket-Origin", None)
-
-        # If there was an origin header, check to make sure it matches
-        # according to check_origin. When the origin is None, we assume it
-        # did not come from a browser and that it can be passed on.
-        if origin is not None and not self.check_origin(origin):
-            self.set_status(403)
-            log_msg = "Cross origin websockets not allowed"
-            self.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        self.ws_connection = self.get_websocket_protocol()
-        if self.ws_connection:
-            self.ws_connection.accept_connection()
-        else:
-            self.set_status(426, "Upgrade Required")
-            self.set_header("Sec-WebSocket-Version", "7, 8, 13")
-            self.finish()
-
-    stream = None
-
-    @property
-    def ping_interval(self):
-        """The interval for websocket keep-alive pings.
-
-        Set websocket_ping_interval = 0 to disable pings.
-        """
-        return self.settings.get('websocket_ping_interval', None)
-
-    @property
-    def ping_timeout(self):
-        """If no ping is received in this many seconds,
-        close the websocket connection (VPNs, etc. can fail to cleanly close ws connections).
-        Default is max of 3 pings or 30 seconds.
-        """
-        return self.settings.get('websocket_ping_timeout', None)
-
-    @property
-    def max_message_size(self):
-        """Maximum allowed message size.
-
-        If the remote peer sends a message larger than this, the connection
-        will be closed.
-
-        Default is 10MiB.
-        """
-        return self.settings.get('websocket_max_message_size', _default_max_message_size)
-
-    def write_message(self, message, binary=False):
-        """Sends the given message to the client of this Web Socket.
-
-        The message may be either a string or a dict (which will be
-        encoded as json).  If the ``binary`` argument is false, the
-        message will be sent as utf8; in binary mode any byte string
-        is allowed.
-
-        If the connection is already closed, raises `WebSocketClosedError`.
-        Returns a `.Future` which can be used for flow control.
-
-        .. versionchanged:: 3.2
-           `WebSocketClosedError` was added (previously a closed connection
-           would raise an `AttributeError`)
-
-        .. versionchanged:: 4.3
-           Returns a `.Future` which can be used for flow control.
-
-        .. versionchanged:: 5.0
-           Consistently raises `WebSocketClosedError`. Previously could
-           sometimes raise `.StreamClosedError`.
-        """
-        if self.ws_connection is None:
-            raise WebSocketClosedError()
-        if isinstance(message, dict):
-            message = tornado.escape.json_encode(message)
-        return self.ws_connection.write_message(message, binary=binary)
-
-    def select_subprotocol(self, subprotocols):
-        """Override to implement subprotocol negotiation.
-
-        ``subprotocols`` is a list of strings identifying the
-        subprotocols proposed by the client.  This method may be
-        overridden to return one of those strings to select it, or
-        ``None`` to not select a subprotocol.
-
-        Failure to select a subprotocol does not automatically abort
-        the connection, although clients may close the connection if
-        none of their proposed subprotocols was selected.
-
-        The list may be empty, in which case this method must return
-        None. This method is always called exactly once even if no
-        subprotocols were proposed so that the handler can be advised
-        of this fact.
-
-        .. versionchanged:: 5.1
-
-           Previously, this method was called with a list containing
-           an empty string instead of an empty list if no subprotocols
-           were proposed by the client.
-        """
-        return None
-
-    @property
-    def selected_subprotocol(self):
-        """The subprotocol returned by `select_subprotocol`.
-
-        .. versionadded:: 5.1
-        """
-        return self.ws_connection.selected_subprotocol
-
-    def get_compression_options(self):
-        """Override to return compression options for the connection.
-
-        If this method returns None (the default), compression will
-        be disabled.  If it returns a dict (even an empty one), it
-        will be enabled.  The contents of the dict may be used to
-        control the following compression options:
-
-        ``compression_level`` specifies the compression level.
-
-        ``mem_level`` specifies the amount of memory used for the internal compression state.
-
-         These parameters are documented in details here:
-         https://docs.python.org/3.6/library/zlib.html#zlib.compressobj
-
-        .. versionadded:: 4.1
-
-        .. versionchanged:: 4.5
-
-           Added ``compression_level`` and ``mem_level``.
-        """
-        # TODO: Add wbits option.
-        return None
-
-    def open(self, *args, **kwargs):
-        """Invoked when a new WebSocket is opened.
-
-        The arguments to `open` are extracted from the `tornado.web.URLSpec`
-        regular expression, just like the arguments to
-        `tornado.web.RequestHandler.get`.
-
-        `open` may be a coroutine. `on_message` will not be called until
-        `open` has returned.
-
-        .. versionchanged:: 5.1
-
-           ``open`` may be a coroutine.
-        """
-        pass
-
-    def on_message(self, message):
-        """Handle incoming messages on the WebSocket
-
-        This method must be overridden.
-
-        .. versionchanged:: 4.5
-
-           ``on_message`` can be a coroutine.
-        """
-        raise NotImplementedError
-
-    def ping(self, data=b''):
-        """Send ping frame to the remote end.
-
-        The data argument allows a small amount of data (up to 125
-        bytes) to be sent as a part of the ping message. Note that not
-        all websocket implementations expose this data to
-        applications.
-
-        Consider using the ``websocket_ping_interval`` application
-        setting instead of sending pings manually.
-
-        .. versionchanged:: 5.1
-
-           The data argument is now optional.
-
-        """
-        data = utf8(data)
-        if self.ws_connection is None:
-            raise WebSocketClosedError()
-        self.ws_connection.write_ping(data)
-
-    def on_pong(self, data):
-        """Invoked when the response to a ping frame is received."""
-        pass
-
-    def on_ping(self, data):
-        """Invoked when the a ping frame is received."""
-        pass
-
-    def on_close(self):
-        """Invoked when the WebSocket is closed.
-
-        If the connection was closed cleanly and a status code or reason
-        phrase was supplied, these values will be available as the attributes
-        ``self.close_code`` and ``self.close_reason``.
-
-        .. versionchanged:: 4.0
-
-           Added ``close_code`` and ``close_reason`` attributes.
-        """
-        pass
-
-    def close(self, code=None, reason=None):
-        """Closes this Web Socket.
-
-        Once the close handshake is successful the socket will be closed.
-
-        ``code`` may be a numeric status code, taken from the values
-        defined in `RFC 6455 section 7.4.1
-        `_.
-        ``reason`` may be a textual message about why the connection is
-        closing.  These values are made available to the client, but are
-        not otherwise interpreted by the websocket protocol.
-
-        .. versionchanged:: 4.0
-
-           Added the ``code`` and ``reason`` arguments.
-        """
-        if self.ws_connection:
-            self.ws_connection.close(code, reason)
-            self.ws_connection = None
-
-    def check_origin(self, origin):
-        """Override to enable support for allowing alternate origins.
-
-        The ``origin`` argument is the value of the ``Origin`` HTTP
-        header, the url responsible for initiating this request.  This
-        method is not called for clients that do not send this header;
-        such requests are always allowed (because all browsers that
-        implement WebSockets support this header, and non-browser
-        clients do not have the same cross-site security concerns).
-
-        Should return True to accept the request or False to reject it.
-        By default, rejects all requests with an origin on a host other
-        than this one.
-
-        This is a security protection against cross site scripting attacks on
-        browsers, since WebSockets are allowed to bypass the usual same-origin
-        policies and don't use CORS headers.
-
-        .. warning::
-
-           This is an important security measure; don't disable it
-           without understanding the security implications. In
-           particular, if your authentication is cookie-based, you
-           must either restrict the origins allowed by
-           ``check_origin()`` or implement your own XSRF-like
-           protection for websocket connections. See `these
-           `_
-           `articles
-           `_
-           for more.
-
-        To accept all cross-origin traffic (which was the default prior to
-        Tornado 4.0), simply override this method to always return true::
-
-            def check_origin(self, origin):
-                return True
-
-        To allow connections from any subdomain of your site, you might
-        do something like::
-
-            def check_origin(self, origin):
-                parsed_origin = urllib.parse.urlparse(origin)
-                return parsed_origin.netloc.endswith(".mydomain.com")
-
-        .. versionadded:: 4.0
-
-        """
-        parsed_origin = urlparse(origin)
-        origin = parsed_origin.netloc
-        origin = origin.lower()
-
-        host = self.request.headers.get("Host")
-
-        # Check to see that origin matches host directly, including ports
-        return origin == host
-
-    def set_nodelay(self, value):
-        """Set the no-delay flag for this stream.
-
-        By default, small messages may be delayed and/or combined to minimize
-        the number of packets sent.  This can sometimes cause 200-500ms delays
-        due to the interaction between Nagle's algorithm and TCP delayed
-        ACKs.  To reduce this delay (at the expense of possibly increasing
-        bandwidth usage), call ``self.set_nodelay(True)`` once the websocket
-        connection is established.
-
-        See `.BaseIOStream.set_nodelay` for additional details.
-
-        .. versionadded:: 3.1
-        """
-        self.stream.set_nodelay(value)
-
-    def on_connection_close(self):
-        if self.ws_connection:
-            self.ws_connection.on_connection_close()
-            self.ws_connection = None
-        if not self._on_close_called:
-            self._on_close_called = True
-            self.on_close()
-            self._break_cycles()
-
-    def _break_cycles(self):
-        # WebSocketHandlers call finish() early, but we don't want to
-        # break up reference cycles (which makes it impossible to call
-        # self.render_string) until after we've really closed the
-        # connection (if it was established in the first place,
-        # indicated by status code 101).
-        if self.get_status() != 101 or self._on_close_called:
-            super(WebSocketHandler, self)._break_cycles()
-
-    def send_error(self, *args, **kwargs):
-        if self.stream is None:
-            super(WebSocketHandler, self).send_error(*args, **kwargs)
-        else:
-            # If we get an uncaught exception during the handshake,
-            # we have no choice but to abruptly close the connection.
-            # TODO: for uncaught exceptions after the handshake,
-            # we can close the connection more gracefully.
-            self.stream.close()
-
-    def get_websocket_protocol(self):
-        websocket_version = self.request.headers.get("Sec-WebSocket-Version")
-        if websocket_version in ("7", "8", "13"):
-            return WebSocketProtocol13(
-                self, compression_options=self.get_compression_options())
-
-    def _attach_stream(self):
-        self.stream = self.detach()
-        self.stream.set_close_callback(self.on_connection_close)
-        # disable non-WS methods
-        for method in ["write", "redirect", "set_header", "set_cookie",
-                       "set_status", "flush", "finish"]:
-            setattr(self, method, _raise_not_supported_for_websockets)
-
-
-def _raise_not_supported_for_websockets(*args, **kwargs):
-    raise RuntimeError("Method not supported for Web Sockets")
-
-
-class WebSocketProtocol(object):
-    """Base class for WebSocket protocol versions.
-    """
-    def __init__(self, handler):
-        self.handler = handler
-        self.request = handler.request
-        self.stream = handler.stream
-        self.client_terminated = False
-        self.server_terminated = False
-
-    def _run_callback(self, callback, *args, **kwargs):
-        """Runs the given callback with exception handling.
-
-        If the callback is a coroutine, returns its Future. On error, aborts the
-        websocket connection and returns None.
-        """
-        try:
-            result = callback(*args, **kwargs)
-        except Exception:
-            self.handler.log_exception(*sys.exc_info())
-            self._abort()
-        else:
-            if result is not None:
-                result = gen.convert_yielded(result)
-                self.stream.io_loop.add_future(result, lambda f: f.result())
-            return result
-
-    def on_connection_close(self):
-        self._abort()
-
-    def _abort(self):
-        """Instantly aborts the WebSocket connection by closing the socket"""
-        self.client_terminated = True
-        self.server_terminated = True
-        self.stream.close()  # forcibly tear down the connection
-        self.close()  # let the subclass cleanup
-
-
-class _PerMessageDeflateCompressor(object):
-    def __init__(self, persistent, max_wbits, compression_options=None):
-        if max_wbits is None:
-            max_wbits = zlib.MAX_WBITS
-        # There is no symbolic constant for the minimum wbits value.
-        if not (8 <= max_wbits <= zlib.MAX_WBITS):
-            raise ValueError("Invalid max_wbits value %r; allowed range 8-%d",
-                             max_wbits, zlib.MAX_WBITS)
-        self._max_wbits = max_wbits
-
-        if compression_options is None or 'compression_level' not in compression_options:
-            self._compression_level = tornado.web.GZipContentEncoding.GZIP_LEVEL
-        else:
-            self._compression_level = compression_options['compression_level']
-
-        if compression_options is None or 'mem_level' not in compression_options:
-            self._mem_level = 8
-        else:
-            self._mem_level = compression_options['mem_level']
-
-        if persistent:
-            self._compressor = self._create_compressor()
-        else:
-            self._compressor = None
-
-    def _create_compressor(self):
-        return zlib.compressobj(self._compression_level,
-                                zlib.DEFLATED, -self._max_wbits, self._mem_level)
-
-    def compress(self, data):
-        compressor = self._compressor or self._create_compressor()
-        data = (compressor.compress(data) +
-                compressor.flush(zlib.Z_SYNC_FLUSH))
-        assert data.endswith(b'\x00\x00\xff\xff')
-        return data[:-4]
-
-
-class _PerMessageDeflateDecompressor(object):
-    def __init__(self, persistent, max_wbits, max_message_size, compression_options=None):
-        self._max_message_size = max_message_size
-        if max_wbits is None:
-            max_wbits = zlib.MAX_WBITS
-        if not (8 <= max_wbits <= zlib.MAX_WBITS):
-            raise ValueError("Invalid max_wbits value %r; allowed range 8-%d",
-                             max_wbits, zlib.MAX_WBITS)
-        self._max_wbits = max_wbits
-        if persistent:
-            self._decompressor = self._create_decompressor()
-        else:
-            self._decompressor = None
-
-    def _create_decompressor(self):
-        return zlib.decompressobj(-self._max_wbits)
-
-    def decompress(self, data):
-        decompressor = self._decompressor or self._create_decompressor()
-        result = decompressor.decompress(data + b'\x00\x00\xff\xff', self._max_message_size)
-        if decompressor.unconsumed_tail:
-            raise _DecompressTooLargeError()
-        return result
-
-
-class WebSocketProtocol13(WebSocketProtocol):
-    """Implementation of the WebSocket protocol from RFC 6455.
-
-    This class supports versions 7 and 8 of the protocol in addition to the
-    final version 13.
-    """
-    # Bit masks for the first byte of a frame.
-    FIN = 0x80
-    RSV1 = 0x40
-    RSV2 = 0x20
-    RSV3 = 0x10
-    RSV_MASK = RSV1 | RSV2 | RSV3
-    OPCODE_MASK = 0x0f
-
-    def __init__(self, handler, mask_outgoing=False,
-                 compression_options=None):
-        WebSocketProtocol.__init__(self, handler)
-        self.mask_outgoing = mask_outgoing
-        self._final_frame = False
-        self._frame_opcode = None
-        self._masked_frame = None
-        self._frame_mask = None
-        self._frame_length = None
-        self._fragmented_message_buffer = None
-        self._fragmented_message_opcode = None
-        self._waiting = None
-        self._compression_options = compression_options
-        self._decompressor = None
-        self._compressor = None
-        self._frame_compressed = None
-        # The total uncompressed size of all messages received or sent.
-        # Unicode messages are encoded to utf8.
-        # Only for testing; subject to change.
-        self._message_bytes_in = 0
-        self._message_bytes_out = 0
-        # The total size of all packets received or sent.  Includes
-        # the effect of compression, frame overhead, and control frames.
-        self._wire_bytes_in = 0
-        self._wire_bytes_out = 0
-        self.ping_callback = None
-        self.last_ping = 0
-        self.last_pong = 0
-
-    def accept_connection(self):
-        try:
-            self._handle_websocket_headers()
-        except ValueError:
-            self.handler.set_status(400)
-            log_msg = "Missing/Invalid WebSocket headers"
-            self.handler.finish(log_msg)
-            gen_log.debug(log_msg)
-            return
-
-        try:
-            self._accept_connection()
-        except ValueError:
-            gen_log.debug("Malformed WebSocket request received",
-                          exc_info=True)
-            self._abort()
-            return
-
-    def _handle_websocket_headers(self):
-        """Verifies all invariant- and required headers
-
-        If a header is missing or have an incorrect value ValueError will be
-        raised
-        """
-        fields = ("Host", "Sec-Websocket-Key", "Sec-Websocket-Version")
-        if not all(map(lambda f: self.request.headers.get(f), fields)):
-            raise ValueError("Missing/Invalid WebSocket headers")
-
-    @staticmethod
-    def compute_accept_value(key):
-        """Computes the value for the Sec-WebSocket-Accept header,
-        given the value for Sec-WebSocket-Key.
-        """
-        sha1 = hashlib.sha1()
-        sha1.update(utf8(key))
-        sha1.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11")  # Magic value
-        return native_str(base64.b64encode(sha1.digest()))
-
-    def _challenge_response(self):
-        return WebSocketProtocol13.compute_accept_value(
-            self.request.headers.get("Sec-Websocket-Key"))
-
-    @gen.coroutine
-    def _accept_connection(self):
-        subprotocol_header = self.request.headers.get("Sec-WebSocket-Protocol")
-        if subprotocol_header:
-            subprotocols = [s.strip() for s in subprotocol_header.split(',')]
-        else:
-            subprotocols = []
-        self.selected_subprotocol = self.handler.select_subprotocol(subprotocols)
-        if self.selected_subprotocol:
-            assert self.selected_subprotocol in subprotocols
-            self.handler.set_header("Sec-WebSocket-Protocol", self.selected_subprotocol)
-
-        extensions = self._parse_extensions_header(self.request.headers)
-        for ext in extensions:
-            if (ext[0] == 'permessage-deflate' and
-                    self._compression_options is not None):
-                # TODO: negotiate parameters if compression_options
-                # specifies limits.
-                self._create_compressors('server', ext[1], self._compression_options)
-                if ('client_max_window_bits' in ext[1] and
-                        ext[1]['client_max_window_bits'] is None):
-                    # Don't echo an offered client_max_window_bits
-                    # parameter with no value.
-                    del ext[1]['client_max_window_bits']
-                self.handler.set_header("Sec-WebSocket-Extensions",
-                                        httputil._encode_header(
-                                            'permessage-deflate', ext[1]))
-                break
-
-        self.handler.clear_header("Content-Type")
-        self.handler.set_status(101)
-        self.handler.set_header("Upgrade", "websocket")
-        self.handler.set_header("Connection", "Upgrade")
-        self.handler.set_header("Sec-WebSocket-Accept", self._challenge_response())
-        self.handler.finish()
-
-        self.handler._attach_stream()
-        self.stream = self.handler.stream
-
-        self.start_pinging()
-        try:
-            open_result = self.handler.open(*self.handler.open_args, **self.handler.open_kwargs)
-            if open_result is not None:
-                yield open_result
-        except Exception:
-            self.handler.log_exception(*sys.exc_info())
-            self._abort()
-
-        yield self._receive_frame_loop()
-
-    def _parse_extensions_header(self, headers):
-        extensions = headers.get("Sec-WebSocket-Extensions", '')
-        if extensions:
-            return [httputil._parse_header(e.strip())
-                    for e in extensions.split(',')]
-        return []
-
-    def _process_server_headers(self, key, headers):
-        """Process the headers sent by the server to this client connection.
-
-        'key' is the websocket handshake challenge/response key.
-        """
-        assert headers['Upgrade'].lower() == 'websocket'
-        assert headers['Connection'].lower() == 'upgrade'
-        accept = self.compute_accept_value(key)
-        assert headers['Sec-Websocket-Accept'] == accept
-
-        extensions = self._parse_extensions_header(headers)
-        for ext in extensions:
-            if (ext[0] == 'permessage-deflate' and
-                    self._compression_options is not None):
-                self._create_compressors('client', ext[1])
-            else:
-                raise ValueError("unsupported extension %r", ext)
-
-        self.selected_subprotocol = headers.get('Sec-WebSocket-Protocol', None)
-
-    def _get_compressor_options(self, side, agreed_parameters, compression_options=None):
-        """Converts a websocket agreed_parameters set to keyword arguments
-        for our compressor objects.
-        """
-        options = dict(
-            persistent=(side + '_no_context_takeover') not in agreed_parameters)
-        wbits_header = agreed_parameters.get(side + '_max_window_bits', None)
-        if wbits_header is None:
-            options['max_wbits'] = zlib.MAX_WBITS
-        else:
-            options['max_wbits'] = int(wbits_header)
-        options['compression_options'] = compression_options
-        return options
-
-    def _create_compressors(self, side, agreed_parameters, compression_options=None):
-        # TODO: handle invalid parameters gracefully
-        allowed_keys = set(['server_no_context_takeover',
-                            'client_no_context_takeover',
-                            'server_max_window_bits',
-                            'client_max_window_bits'])
-        for key in agreed_parameters:
-            if key not in allowed_keys:
-                raise ValueError("unsupported compression parameter %r" % key)
-        other_side = 'client' if (side == 'server') else 'server'
-        self._compressor = _PerMessageDeflateCompressor(
-            **self._get_compressor_options(side, agreed_parameters, compression_options))
-        self._decompressor = _PerMessageDeflateDecompressor(
-            max_message_size=self.handler.max_message_size,
-            **self._get_compressor_options(other_side, agreed_parameters, compression_options))
-
-    def _write_frame(self, fin, opcode, data, flags=0):
-        data_len = len(data)
-        if opcode & 0x8:
-            # All control frames MUST have a payload length of 125
-            # bytes or less and MUST NOT be fragmented.
-            if not fin:
-                raise ValueError("control frames may not be fragmented")
-            if data_len > 125:
-                raise ValueError("control frame payloads may not exceed 125 bytes")
-        if fin:
-            finbit = self.FIN
-        else:
-            finbit = 0
-        frame = struct.pack("B", finbit | opcode | flags)
-        if self.mask_outgoing:
-            mask_bit = 0x80
-        else:
-            mask_bit = 0
-        if data_len < 126:
-            frame += struct.pack("B", data_len | mask_bit)
-        elif data_len <= 0xFFFF:
-            frame += struct.pack("!BH", 126 | mask_bit, data_len)
-        else:
-            frame += struct.pack("!BQ", 127 | mask_bit, data_len)
-        if self.mask_outgoing:
-            mask = os.urandom(4)
-            data = mask + _websocket_mask(mask, data)
-        frame += data
-        self._wire_bytes_out += len(frame)
-        return self.stream.write(frame)
-
-    def write_message(self, message, binary=False):
-        """Sends the given message to the client of this Web Socket."""
-        if binary:
-            opcode = 0x2
-        else:
-            opcode = 0x1
-        message = tornado.escape.utf8(message)
-        assert isinstance(message, bytes)
-        self._message_bytes_out += len(message)
-        flags = 0
-        if self._compressor:
-            message = self._compressor.compress(message)
-            flags |= self.RSV1
-        # For historical reasons, write methods in Tornado operate in a semi-synchronous
-        # mode in which awaiting the Future they return is optional (But errors can
-        # still be raised). This requires us to go through an awkward dance here
-        # to transform the errors that may be returned while presenting the same
-        # semi-synchronous interface.
-        try:
-            fut = self._write_frame(True, opcode, message, flags=flags)
-        except StreamClosedError:
-            raise WebSocketClosedError()
-
-        @gen.coroutine
-        def wrapper():
-            try:
-                yield fut
-            except StreamClosedError:
-                raise WebSocketClosedError()
-        return wrapper()
-
-    def write_ping(self, data):
-        """Send ping frame."""
-        assert isinstance(data, bytes)
-        self._write_frame(True, 0x9, data)
-
-    @gen.coroutine
-    def _receive_frame_loop(self):
-        try:
-            while not self.client_terminated:
-                yield self._receive_frame()
-        except StreamClosedError:
-            self._abort()
-
-    def _read_bytes(self, n):
-        self._wire_bytes_in += n
-        return self.stream.read_bytes(n)
-
-    @gen.coroutine
-    def _receive_frame(self):
-        # Read the frame header.
-        data = yield self._read_bytes(2)
-        header, mask_payloadlen = struct.unpack("BB", data)
-        is_final_frame = header & self.FIN
-        reserved_bits = header & self.RSV_MASK
-        opcode = header & self.OPCODE_MASK
-        opcode_is_control = opcode & 0x8
-        if self._decompressor is not None and opcode != 0:
-            # Compression flag is present in the first frame's header,
-            # but we can't decompress until we have all the frames of
-            # the message.
-            self._frame_compressed = bool(reserved_bits & self.RSV1)
-            reserved_bits &= ~self.RSV1
-        if reserved_bits:
-            # client is using as-yet-undefined extensions; abort
-            self._abort()
-            return
-        is_masked = bool(mask_payloadlen & 0x80)
-        payloadlen = mask_payloadlen & 0x7f
-
-        # Parse and validate the length.
-        if opcode_is_control and payloadlen >= 126:
-            # control frames must have payload < 126
-            self._abort()
-            return
-        if payloadlen < 126:
-            self._frame_length = payloadlen
-        elif payloadlen == 126:
-            data = yield self._read_bytes(2)
-            payloadlen = struct.unpack("!H", data)[0]
-        elif payloadlen == 127:
-            data = yield self._read_bytes(8)
-            payloadlen = struct.unpack("!Q", data)[0]
-        new_len = payloadlen
-        if self._fragmented_message_buffer is not None:
-            new_len += len(self._fragmented_message_buffer)
-        if new_len > self.handler.max_message_size:
-            self.close(1009, "message too big")
-            self._abort()
-            return
-
-        # Read the payload, unmasking if necessary.
-        if is_masked:
-            self._frame_mask = yield self._read_bytes(4)
-        data = yield self._read_bytes(payloadlen)
-        if is_masked:
-            data = _websocket_mask(self._frame_mask, data)
-
-        # Decide what to do with this frame.
-        if opcode_is_control:
-            # control frames may be interleaved with a series of fragmented
-            # data frames, so control frames must not interact with
-            # self._fragmented_*
-            if not is_final_frame:
-                # control frames must not be fragmented
-                self._abort()
-                return
-        elif opcode == 0:  # continuation frame
-            if self._fragmented_message_buffer is None:
-                # nothing to continue
-                self._abort()
-                return
-            self._fragmented_message_buffer += data
-            if is_final_frame:
-                opcode = self._fragmented_message_opcode
-                data = self._fragmented_message_buffer
-                self._fragmented_message_buffer = None
-        else:  # start of new data message
-            if self._fragmented_message_buffer is not None:
-                # can't start new message until the old one is finished
-                self._abort()
-                return
-            if not is_final_frame:
-                self._fragmented_message_opcode = opcode
-                self._fragmented_message_buffer = data
-
-        if is_final_frame:
-            handled_future = self._handle_message(opcode, data)
-            if handled_future is not None:
-                yield handled_future
-
-    def _handle_message(self, opcode, data):
-        """Execute on_message, returning its Future if it is a coroutine."""
-        if self.client_terminated:
-            return
-
-        if self._frame_compressed:
-            try:
-                data = self._decompressor.decompress(data)
-            except _DecompressTooLargeError:
-                self.close(1009, "message too big after decompression")
-                self._abort()
-                return
-
-        if opcode == 0x1:
-            # UTF-8 data
-            self._message_bytes_in += len(data)
-            try:
-                decoded = data.decode("utf-8")
-            except UnicodeDecodeError:
-                self._abort()
-                return
-            return self._run_callback(self.handler.on_message, decoded)
-        elif opcode == 0x2:
-            # Binary data
-            self._message_bytes_in += len(data)
-            return self._run_callback(self.handler.on_message, data)
-        elif opcode == 0x8:
-            # Close
-            self.client_terminated = True
-            if len(data) >= 2:
-                self.handler.close_code = struct.unpack('>H', data[:2])[0]
-            if len(data) > 2:
-                self.handler.close_reason = to_unicode(data[2:])
-            # Echo the received close code, if any (RFC 6455 section 5.5.1).
-            self.close(self.handler.close_code)
-        elif opcode == 0x9:
-            # Ping
-            try:
-                self._write_frame(True, 0xA, data)
-            except StreamClosedError:
-                self._abort()
-            self._run_callback(self.handler.on_ping, data)
-        elif opcode == 0xA:
-            # Pong
-            self.last_pong = IOLoop.current().time()
-            return self._run_callback(self.handler.on_pong, data)
-        else:
-            self._abort()
-
-    def close(self, code=None, reason=None):
-        """Closes the WebSocket connection."""
-        if not self.server_terminated:
-            if not self.stream.closed():
-                if code is None and reason is not None:
-                    code = 1000  # "normal closure" status code
-                if code is None:
-                    close_data = b''
-                else:
-                    close_data = struct.pack('>H', code)
-                if reason is not None:
-                    close_data += utf8(reason)
-                try:
-                    self._write_frame(True, 0x8, close_data)
-                except StreamClosedError:
-                    self._abort()
-            self.server_terminated = True
-        if self.client_terminated:
-            if self._waiting is not None:
-                self.stream.io_loop.remove_timeout(self._waiting)
-                self._waiting = None
-            self.stream.close()
-        elif self._waiting is None:
-            # Give the client a few seconds to complete a clean shutdown,
-            # otherwise just close the connection.
-            self._waiting = self.stream.io_loop.add_timeout(
-                self.stream.io_loop.time() + 5, self._abort)
-
-    @property
-    def ping_interval(self):
-        interval = self.handler.ping_interval
-        if interval is not None:
-            return interval
-        return 0
-
-    @property
-    def ping_timeout(self):
-        timeout = self.handler.ping_timeout
-        if timeout is not None:
-            return timeout
-        return max(3 * self.ping_interval, 30)
-
-    def start_pinging(self):
-        """Start sending periodic pings to keep the connection alive"""
-        if self.ping_interval > 0:
-            self.last_ping = self.last_pong = IOLoop.current().time()
-            self.ping_callback = PeriodicCallback(
-                self.periodic_ping, self.ping_interval * 1000)
-            self.ping_callback.start()
-
-    def periodic_ping(self):
-        """Send a ping to keep the websocket alive
-
-        Called periodically if the websocket_ping_interval is set and non-zero.
-        """
-        if self.stream.closed() and self.ping_callback is not None:
-            self.ping_callback.stop()
-            return
-
-        # Check for timeout on pong. Make sure that we really have
-        # sent a recent ping in case the machine with both server and
-        # client has been suspended since the last ping.
-        now = IOLoop.current().time()
-        since_last_pong = now - self.last_pong
-        since_last_ping = now - self.last_ping
-        if (since_last_ping < 2 * self.ping_interval and
-                since_last_pong > self.ping_timeout):
-            self.close()
-            return
-
-        self.write_ping(b'')
-        self.last_ping = now
-
-
-class WebSocketClientConnection(simple_httpclient._HTTPConnection):
-    """WebSocket client connection.
-
-    This class should not be instantiated directly; use the
-    `websocket_connect` function instead.
-    """
-    def __init__(self, request, on_message_callback=None,
-                 compression_options=None, ping_interval=None, ping_timeout=None,
-                 max_message_size=None, subprotocols=[]):
-        self.compression_options = compression_options
-        self.connect_future = Future()
-        self.protocol = None
-        self.read_queue = Queue(1)
-        self.key = base64.b64encode(os.urandom(16))
-        self._on_message_callback = on_message_callback
-        self.close_code = self.close_reason = None
-        self.ping_interval = ping_interval
-        self.ping_timeout = ping_timeout
-        self.max_message_size = max_message_size
-
-        scheme, sep, rest = request.url.partition(':')
-        scheme = {'ws': 'http', 'wss': 'https'}[scheme]
-        request.url = scheme + sep + rest
-        request.headers.update({
-            'Upgrade': 'websocket',
-            'Connection': 'Upgrade',
-            'Sec-WebSocket-Key': self.key,
-            'Sec-WebSocket-Version': '13',
-        })
-        if subprotocols is not None:
-            request.headers['Sec-WebSocket-Protocol'] = ','.join(subprotocols)
-        if self.compression_options is not None:
-            # Always offer to let the server set our max_wbits (and even though
-            # we don't offer it, we will accept a client_no_context_takeover
-            # from the server).
-            # TODO: set server parameters for deflate extension
-            # if requested in self.compression_options.
-            request.headers['Sec-WebSocket-Extensions'] = (
-                'permessage-deflate; client_max_window_bits')
-
-        self.tcp_client = TCPClient()
-        super(WebSocketClientConnection, self).__init__(
-            None, request, lambda: None, self._on_http_response,
-            104857600, self.tcp_client, 65536, 104857600)
-
-    def close(self, code=None, reason=None):
-        """Closes the websocket connection.
-
-        ``code`` and ``reason`` are documented under
-        `WebSocketHandler.close`.
-
-        .. versionadded:: 3.2
-
-        .. versionchanged:: 4.0
-
-           Added the ``code`` and ``reason`` arguments.
-        """
-        if self.protocol is not None:
-            self.protocol.close(code, reason)
-            self.protocol = None
-
-    def on_connection_close(self):
-        if not self.connect_future.done():
-            self.connect_future.set_exception(StreamClosedError())
-        self.on_message(None)
-        self.tcp_client.close()
-        super(WebSocketClientConnection, self).on_connection_close()
-
-    def _on_http_response(self, response):
-        if not self.connect_future.done():
-            if response.error:
-                self.connect_future.set_exception(response.error)
-            else:
-                self.connect_future.set_exception(WebSocketError(
-                    "Non-websocket response"))
-
-    def headers_received(self, start_line, headers):
-        if start_line.code != 101:
-            return super(WebSocketClientConnection, self).headers_received(
-                start_line, headers)
-
-        self.headers = headers
-        self.protocol = self.get_websocket_protocol()
-        self.protocol._process_server_headers(self.key, self.headers)
-        self.protocol.start_pinging()
-        IOLoop.current().add_callback(self.protocol._receive_frame_loop)
-
-        if self._timeout is not None:
-            self.io_loop.remove_timeout(self._timeout)
-            self._timeout = None
-
-        self.stream = self.connection.detach()
-        self.stream.set_close_callback(self.on_connection_close)
-        # Once we've taken over the connection, clear the final callback
-        # we set on the http request.  This deactivates the error handling
-        # in simple_httpclient that would otherwise interfere with our
-        # ability to see exceptions.
-        self.final_callback = None
-
-        future_set_result_unless_cancelled(self.connect_future, self)
-
-    def write_message(self, message, binary=False):
-        """Sends a message to the WebSocket server.
-
-        If the stream is closed, raises `WebSocketClosedError`.
-        Returns a `.Future` which can be used for flow control.
-
-        .. versionchanged:: 5.0
-           Exception raised on a closed stream changed from `.StreamClosedError`
-           to `WebSocketClosedError`.
-        """
-        return self.protocol.write_message(message, binary=binary)
-
-    def read_message(self, callback=None):
-        """Reads a message from the WebSocket server.
-
-        If on_message_callback was specified at WebSocket
-        initialization, this function will never return messages
-
-        Returns a future whose result is the message, or None
-        if the connection is closed.  If a callback argument
-        is given it will be called with the future when it is
-        ready.
-        """
-
-        future = self.read_queue.get()
-        if callback is not None:
-            self.io_loop.add_future(future, callback)
-        return future
-
-    def on_message(self, message):
-        if self._on_message_callback:
-            self._on_message_callback(message)
-        else:
-            return self.read_queue.put(message)
-
-    def ping(self, data=b''):
-        """Send ping frame to the remote end.
-
-        The data argument allows a small amount of data (up to 125
-        bytes) to be sent as a part of the ping message. Note that not
-        all websocket implementations expose this data to
-        applications.
-
-        Consider using the ``ping_interval`` argument to
-        `websocket_connect` instead of sending pings manually.
-
-        .. versionadded:: 5.1
-
-        """
-        data = utf8(data)
-        if self.protocol is None:
-            raise WebSocketClosedError()
-        self.protocol.write_ping(data)
-
-    def on_pong(self, data):
-        pass
-
-    def on_ping(self, data):
-        pass
-
-    def get_websocket_protocol(self):
-        return WebSocketProtocol13(self, mask_outgoing=True,
-                                   compression_options=self.compression_options)
-
-    @property
-    def selected_subprotocol(self):
-        """The subprotocol selected by the server.
-
-        .. versionadded:: 5.1
-        """
-        return self.protocol.selected_subprotocol
-
-
-def websocket_connect(url, callback=None, connect_timeout=None,
-                      on_message_callback=None, compression_options=None,
-                      ping_interval=None, ping_timeout=None,
-                      max_message_size=_default_max_message_size, subprotocols=None):
-    """Client-side websocket support.
-
-    Takes a url and returns a Future whose result is a
-    `WebSocketClientConnection`.
-
-    ``compression_options`` is interpreted in the same way as the
-    return value of `.WebSocketHandler.get_compression_options`.
-
-    The connection supports two styles of operation. In the coroutine
-    style, the application typically calls
-    `~.WebSocketClientConnection.read_message` in a loop::
-
-        conn = yield websocket_connect(url)
-        while True:
-            msg = yield conn.read_message()
-            if msg is None: break
-            # Do something with msg
-
-    In the callback style, pass an ``on_message_callback`` to
-    ``websocket_connect``. In both styles, a message of ``None``
-    indicates that the connection has been closed.
-
-    ``subprotocols`` may be a list of strings specifying proposed
-    subprotocols. The selected protocol may be found on the
-    ``selected_subprotocol`` attribute of the connection object
-    when the connection is complete.
-
-    .. versionchanged:: 3.2
-       Also accepts ``HTTPRequest`` objects in place of urls.
-
-    .. versionchanged:: 4.1
-       Added ``compression_options`` and ``on_message_callback``.
-
-    .. versionchanged:: 4.5
-       Added the ``ping_interval``, ``ping_timeout``, and ``max_message_size``
-       arguments, which have the same meaning as in `WebSocketHandler`.
-
-    .. versionchanged:: 5.0
-       The ``io_loop`` argument (deprecated since version 4.1) has been removed.
-
-    .. versionchanged:: 5.1
-       Added the ``subprotocols`` argument.
-    """
-    if isinstance(url, httpclient.HTTPRequest):
-        assert connect_timeout is None
-        request = url
-        # Copy and convert the headers dict/object (see comments in
-        # AsyncHTTPClient.fetch)
-        request.headers = httputil.HTTPHeaders(request.headers)
-    else:
-        request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout)
-    request = httpclient._RequestProxy(
-        request, httpclient.HTTPRequest._DEFAULTS)
-    conn = WebSocketClientConnection(request,
-                                     on_message_callback=on_message_callback,
-                                     compression_options=compression_options,
-                                     ping_interval=ping_interval,
-                                     ping_timeout=ping_timeout,
-                                     max_message_size=max_message_size,
-                                     subprotocols=subprotocols)
-    if callback is not None:
-        IOLoop.current().add_future(conn.connect_future, callback)
-    return conn.connect_future
diff --git a/lib/tornado/wsgi.py b/lib/tornado/wsgi.py
deleted file mode 100755
index e1230da0..00000000
--- a/lib/tornado/wsgi.py
+++ /dev/null
@@ -1,377 +0,0 @@
-#
-# Copyright 2009 Facebook
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""WSGI support for the Tornado web framework.
-
-WSGI is the Python standard for web servers, and allows for interoperability
-between Tornado and other Python web frameworks and servers.  This module
-provides WSGI support in two ways:
-
-* `WSGIAdapter` converts a `tornado.web.Application` to the WSGI application
-  interface.  This is useful for running a Tornado app on another
-  HTTP server, such as Google App Engine.  See the `WSGIAdapter` class
-  documentation for limitations that apply.
-* `WSGIContainer` lets you run other WSGI applications and frameworks on the
-  Tornado HTTP server.  For example, with this class you can mix Django
-  and Tornado handlers in a single server.
-"""
-
-from __future__ import absolute_import, division, print_function
-
-import sys
-from io import BytesIO
-import tornado
-import warnings
-
-from tornado.concurrent import Future
-from tornado import escape
-from tornado import httputil
-from tornado.log import access_log
-from tornado import web
-from tornado.escape import native_str
-from tornado.util import unicode_type, PY3
-
-
-if PY3:
-    import urllib.parse as urllib_parse  # py3
-else:
-    import urllib as urllib_parse
-
-# PEP 3333 specifies that WSGI on python 3 generally deals with byte strings
-# that are smuggled inside objects of type unicode (via the latin1 encoding).
-# These functions are like those in the tornado.escape module, but defined
-# here to minimize the temptation to use them in non-wsgi contexts.
-if str is unicode_type:
-    def to_wsgi_str(s):
-        assert isinstance(s, bytes)
-        return s.decode('latin1')
-
-    def from_wsgi_str(s):
-        assert isinstance(s, str)
-        return s.encode('latin1')
-else:
-    def to_wsgi_str(s):
-        assert isinstance(s, bytes)
-        return s
-
-    def from_wsgi_str(s):
-        assert isinstance(s, str)
-        return s
-
-
-class WSGIApplication(web.Application):
-    """A WSGI equivalent of `tornado.web.Application`.
-
-    .. deprecated:: 4.0
-
-       Use a regular `.Application` and wrap it in `WSGIAdapter` instead.
-       This class will be removed in Tornado 6.0.
-    """
-    def __call__(self, environ, start_response):
-        return WSGIAdapter(self)(environ, start_response)
-
-
-# WSGI has no facilities for flow control, so just return an already-done
-# Future when the interface requires it.
-def _dummy_future():
-    f = Future()
-    f.set_result(None)
-    return f
-
-
-class _WSGIConnection(httputil.HTTPConnection):
-    def __init__(self, method, start_response, context):
-        self.method = method
-        self.start_response = start_response
-        self.context = context
-        self._write_buffer = []
-        self._finished = False
-        self._expected_content_remaining = None
-        self._error = None
-
-    def set_close_callback(self, callback):
-        # WSGI has no facility for detecting a closed connection mid-request,
-        # so we can simply ignore the callback.
-        pass
-
-    def write_headers(self, start_line, headers, chunk=None, callback=None):
-        if self.method == 'HEAD':
-            self._expected_content_remaining = 0
-        elif 'Content-Length' in headers:
-            self._expected_content_remaining = int(headers['Content-Length'])
-        else:
-            self._expected_content_remaining = None
-        self.start_response(
-            '%s %s' % (start_line.code, start_line.reason),
-            [(native_str(k), native_str(v)) for (k, v) in headers.get_all()])
-        if chunk is not None:
-            self.write(chunk, callback)
-        elif callback is not None:
-            callback()
-        return _dummy_future()
-
-    def write(self, chunk, callback=None):
-        if self._expected_content_remaining is not None:
-            self._expected_content_remaining -= len(chunk)
-            if self._expected_content_remaining < 0:
-                self._error = httputil.HTTPOutputError(
-                    "Tried to write more data than Content-Length")
-                raise self._error
-        self._write_buffer.append(chunk)
-        if callback is not None:
-            callback()
-        return _dummy_future()
-
-    def finish(self):
-        if (self._expected_content_remaining is not None and
-                self._expected_content_remaining != 0):
-            self._error = httputil.HTTPOutputError(
-                "Tried to write %d bytes less than Content-Length" %
-                self._expected_content_remaining)
-            raise self._error
-        self._finished = True
-
-
-class _WSGIRequestContext(object):
-    def __init__(self, remote_ip, protocol):
-        self.remote_ip = remote_ip
-        self.protocol = protocol
-
-    def __str__(self):
-        return self.remote_ip
-
-
-class WSGIAdapter(object):
-    """Converts a `tornado.web.Application` instance into a WSGI application.
-
-    Example usage::
-
-        import tornado.web
-        import tornado.wsgi
-        import wsgiref.simple_server
-
-        class MainHandler(tornado.web.RequestHandler):
-            def get(self):
-                self.write("Hello, world")
-
-        if __name__ == "__main__":
-            application = tornado.web.Application([
-                (r"/", MainHandler),
-            ])
-            wsgi_app = tornado.wsgi.WSGIAdapter(application)
-            server = wsgiref.simple_server.make_server('', 8888, wsgi_app)
-            server.serve_forever()
-
-    See the `appengine demo
-    `_
-    for an example of using this module to run a Tornado app on Google
-    App Engine.
-
-    In WSGI mode asynchronous methods are not supported.  This means
-    that it is not possible to use `.AsyncHTTPClient`, or the
-    `tornado.auth` or `tornado.websocket` modules.
-
-    In multithreaded WSGI servers on Python 3, it may be necessary to
-    permit `asyncio` to create event loops on any thread. Run the
-    following at startup (typically import time for WSGI
-    applications)::
-
-        import asyncio
-        from tornado.platform.asyncio import AnyThreadEventLoopPolicy
-        asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
-
-    .. versionadded:: 4.0
-
-    .. deprecated:: 5.1
-
-       This class is deprecated and will be removed in Tornado 6.0.
-       Use Tornado's `.HTTPServer` instead of a WSGI container.
-    """
-    def __init__(self, application):
-        warnings.warn("WSGIAdapter is deprecated, use Tornado's HTTPServer instead",
-                      DeprecationWarning)
-        if isinstance(application, WSGIApplication):
-            self.application = lambda request: web.Application.__call__(
-                application, request)
-        else:
-            self.application = application
-
-    def __call__(self, environ, start_response):
-        method = environ["REQUEST_METHOD"]
-        uri = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
-        uri += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
-        if environ.get("QUERY_STRING"):
-            uri += "?" + environ["QUERY_STRING"]
-        headers = httputil.HTTPHeaders()
-        if environ.get("CONTENT_TYPE"):
-            headers["Content-Type"] = environ["CONTENT_TYPE"]
-        if environ.get("CONTENT_LENGTH"):
-            headers["Content-Length"] = environ["CONTENT_LENGTH"]
-        for key in environ:
-            if key.startswith("HTTP_"):
-                headers[key[5:].replace("_", "-")] = environ[key]
-        if headers.get("Content-Length"):
-            body = environ["wsgi.input"].read(
-                int(headers["Content-Length"]))
-        else:
-            body = b""
-        protocol = environ["wsgi.url_scheme"]
-        remote_ip = environ.get("REMOTE_ADDR", "")
-        if environ.get("HTTP_HOST"):
-            host = environ["HTTP_HOST"]
-        else:
-            host = environ["SERVER_NAME"]
-        connection = _WSGIConnection(method, start_response,
-                                     _WSGIRequestContext(remote_ip, protocol))
-        request = httputil.HTTPServerRequest(
-            method, uri, "HTTP/1.1", headers=headers, body=body,
-            host=host, connection=connection)
-        request._parse_body()
-        self.application(request)
-        if connection._error:
-            raise connection._error
-        if not connection._finished:
-            raise Exception("request did not finish synchronously")
-        return connection._write_buffer
-
-
-class WSGIContainer(object):
-    r"""Makes a WSGI-compatible function runnable on Tornado's HTTP server.
-
-    .. warning::
-
-       WSGI is a *synchronous* interface, while Tornado's concurrency model
-       is based on single-threaded asynchronous execution.  This means that
-       running a WSGI app with Tornado's `WSGIContainer` is *less scalable*
-       than running the same app in a multi-threaded WSGI server like
-       ``gunicorn`` or ``uwsgi``.  Use `WSGIContainer` only when there are
-       benefits to combining Tornado and WSGI in the same process that
-       outweigh the reduced scalability.
-
-    Wrap a WSGI function in a `WSGIContainer` and pass it to `.HTTPServer` to
-    run it. For example::
-
-        def simple_app(environ, start_response):
-            status = "200 OK"
-            response_headers = [("Content-type", "text/plain")]
-            start_response(status, response_headers)
-            return ["Hello world!\n"]
-
-        container = tornado.wsgi.WSGIContainer(simple_app)
-        http_server = tornado.httpserver.HTTPServer(container)
-        http_server.listen(8888)
-        tornado.ioloop.IOLoop.current().start()
-
-    This class is intended to let other frameworks (Django, web.py, etc)
-    run on the Tornado HTTP server and I/O loop.
-
-    The `tornado.web.FallbackHandler` class is often useful for mixing
-    Tornado and WSGI apps in the same server.  See
-    https://github.com/bdarnell/django-tornado-demo for a complete example.
-    """
-    def __init__(self, wsgi_application):
-        self.wsgi_application = wsgi_application
-
-    def __call__(self, request):
-        data = {}
-        response = []
-
-        def start_response(status, response_headers, exc_info=None):
-            data["status"] = status
-            data["headers"] = response_headers
-            return response.append
-        app_response = self.wsgi_application(
-            WSGIContainer.environ(request), start_response)
-        try:
-            response.extend(app_response)
-            body = b"".join(response)
-        finally:
-            if hasattr(app_response, "close"):
-                app_response.close()
-        if not data:
-            raise Exception("WSGI app did not call start_response")
-
-        status_code, reason = data["status"].split(' ', 1)
-        status_code = int(status_code)
-        headers = data["headers"]
-        header_set = set(k.lower() for (k, v) in headers)
-        body = escape.utf8(body)
-        if status_code != 304:
-            if "content-length" not in header_set:
-                headers.append(("Content-Length", str(len(body))))
-            if "content-type" not in header_set:
-                headers.append(("Content-Type", "text/html; charset=UTF-8"))
-        if "server" not in header_set:
-            headers.append(("Server", "TornadoServer/%s" % tornado.version))
-
-        start_line = httputil.ResponseStartLine("HTTP/1.1", status_code, reason)
-        header_obj = httputil.HTTPHeaders()
-        for key, value in headers:
-            header_obj.add(key, value)
-        request.connection.write_headers(start_line, header_obj, chunk=body)
-        request.connection.finish()
-        self._log(status_code, request)
-
-    @staticmethod
-    def environ(request):
-        """Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment.
-        """
-        hostport = request.host.split(":")
-        if len(hostport) == 2:
-            host = hostport[0]
-            port = int(hostport[1])
-        else:
-            host = request.host
-            port = 443 if request.protocol == "https" else 80
-        environ = {
-            "REQUEST_METHOD": request.method,
-            "SCRIPT_NAME": "",
-            "PATH_INFO": to_wsgi_str(escape.url_unescape(
-                request.path, encoding=None, plus=False)),
-            "QUERY_STRING": request.query,
-            "REMOTE_ADDR": request.remote_ip,
-            "SERVER_NAME": host,
-            "SERVER_PORT": str(port),
-            "SERVER_PROTOCOL": request.version,
-            "wsgi.version": (1, 0),
-            "wsgi.url_scheme": request.protocol,
-            "wsgi.input": BytesIO(escape.utf8(request.body)),
-            "wsgi.errors": sys.stderr,
-            "wsgi.multithread": False,
-            "wsgi.multiprocess": True,
-            "wsgi.run_once": False,
-        }
-        if "Content-Type" in request.headers:
-            environ["CONTENT_TYPE"] = request.headers.pop("Content-Type")
-        if "Content-Length" in request.headers:
-            environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length")
-        for key, value in request.headers.items():
-            environ["HTTP_" + key.replace("-", "_").upper()] = value
-        return environ
-
-    def _log(self, status_code, request):
-        if status_code < 400:
-            log_method = access_log.info
-        elif status_code < 500:
-            log_method = access_log.warning
-        else:
-            log_method = access_log.error
-        request_time = 1000.0 * request.request_time()
-        summary = request.method + " " + request.uri + " (" + \
-            request.remote_ip + ")"
-        log_method("%d %s %.2fms", status_code, summary, request_time)
-
-
-HTTPRequest = httputil.HTTPServerRequest
diff --git a/lib/urllib3/__init__.py b/lib/urllib3/__init__.py
index aaa6b1c6..8f5a21f3 100644
--- a/lib/urllib3/__init__.py
+++ b/lib/urllib3/__init__.py
@@ -1,15 +1,10 @@
 """
 urllib3 - Thread-safe connection pooling and re-using.
 """
-
 from __future__ import absolute_import
 import warnings
 
-from .connectionpool import (
-    HTTPConnectionPool,
-    HTTPSConnectionPool,
-    connection_from_url
-)
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
 
 from . import exceptions
 from .filepost import encode_multipart_formdata
@@ -23,32 +18,27 @@ from .util.retry import Retry
 
 # Set default logging handler to avoid "No handler found" warnings.
 import logging
-try:  # Python 2.7+
-    from logging import NullHandler
-except ImportError:
-    class NullHandler(logging.Handler):
-        def emit(self, record):
-            pass
+from logging import NullHandler
 
-__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
-__license__ = 'MIT'
-__version__ = '1.22'
+__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
+__license__ = "MIT"
+__version__ = "1.25.6"
 
 __all__ = (
-    'HTTPConnectionPool',
-    'HTTPSConnectionPool',
-    'PoolManager',
-    'ProxyManager',
-    'HTTPResponse',
-    'Retry',
-    'Timeout',
-    'add_stderr_logger',
-    'connection_from_url',
-    'disable_warnings',
-    'encode_multipart_formdata',
-    'get_host',
-    'make_headers',
-    'proxy_from_url',
+    "HTTPConnectionPool",
+    "HTTPSConnectionPool",
+    "PoolManager",
+    "ProxyManager",
+    "HTTPResponse",
+    "Retry",
+    "Timeout",
+    "add_stderr_logger",
+    "connection_from_url",
+    "disable_warnings",
+    "encode_multipart_formdata",
+    "get_host",
+    "make_headers",
+    "proxy_from_url",
 )
 
 logging.getLogger(__name__).addHandler(NullHandler())
@@ -65,10 +55,10 @@ def add_stderr_logger(level=logging.DEBUG):
     # even if urllib3 is vendored within another package.
     logger = logging.getLogger(__name__)
     handler = logging.StreamHandler()
-    handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
+    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
     logger.addHandler(handler)
     logger.setLevel(level)
-    logger.debug('Added a stderr logging handler to logger: %s', __name__)
+    logger.debug("Added a stderr logging handler to logger: %s", __name__)
     return handler
 
 
@@ -80,18 +70,17 @@ del NullHandler
 # shouldn't be: otherwise, it's very hard for users to use most Python
 # mechanisms to silence them.
 # SecurityWarning's always go off by default.
-warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
+warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
 # SubjectAltNameWarning's should go off once per host
-warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True)
+warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True)
 # InsecurePlatformWarning's don't vary between requests, so we keep it default.
-warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
-                      append=True)
+warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
 # SNIMissingWarnings should go off only once.
-warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True)
+warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True)
 
 
 def disable_warnings(category=exceptions.HTTPWarning):
     """
     Helper for quickly disabling all urllib3 warnings.
     """
-    warnings.simplefilter('ignore', category)
+    warnings.simplefilter("ignore", category)
diff --git a/lib/urllib3/_collections.py b/lib/urllib3/_collections.py
index 5df2372c..019d1511 100644
--- a/lib/urllib3/_collections.py
+++ b/lib/urllib3/_collections.py
@@ -1,8 +1,13 @@
 from __future__ import absolute_import
-from collections import Mapping, MutableMapping
+
+try:
+    from collections.abc import Mapping, MutableMapping
+except ImportError:
+    from collections import Mapping, MutableMapping
 try:
     from threading import RLock
 except ImportError:  # Platform-specific: No threads available
+
     class RLock:
         def __enter__(self):
             pass
@@ -11,14 +16,12 @@ except ImportError:  # Platform-specific: No threads available
             pass
 
 
-try:  # Python 2.7+
-    from collections import OrderedDict
-except ImportError:
-    from .packages.ordered_dict import OrderedDict
+from collections import OrderedDict
+from .exceptions import InvalidHeader
 from .packages.six import iterkeys, itervalues, PY3
 
 
-__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
+__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
 
 
 _Null = object()
@@ -81,7 +84,9 @@ class RecentlyUsedContainer(MutableMapping):
             return len(self._container)
 
     def __iter__(self):
-        raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
+        raise NotImplementedError(
+            "Iteration over this class is unlikely to be threadsafe."
+        )
 
     def clear(self):
         with self.lock:
@@ -149,7 +154,7 @@ class HTTPHeaderDict(MutableMapping):
 
     def __getitem__(self, key):
         val = self._container[key.lower()]
-        return ', '.join(val[1:])
+        return ", ".join(val[1:])
 
     def __delitem__(self, key):
         del self._container[key.lower()]
@@ -158,12 +163,13 @@ class HTTPHeaderDict(MutableMapping):
         return key.lower() in self._container
 
     def __eq__(self, other):
-        if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
+        if not isinstance(other, Mapping) and not hasattr(other, "keys"):
             return False
         if not isinstance(other, type(self)):
             other = type(self)(other)
-        return (dict((k.lower(), v) for k, v in self.itermerged()) ==
-                dict((k.lower(), v) for k, v in other.itermerged()))
+        return dict((k.lower(), v) for k, v in self.itermerged()) == dict(
+            (k.lower(), v) for k, v in other.itermerged()
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -183,9 +189,9 @@ class HTTPHeaderDict(MutableMapping):
             yield vals[0]
 
     def pop(self, key, default=__marker):
-        '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+        """D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
           If key is not found, d is returned if given, otherwise KeyError is raised.
-        '''
+        """
         # Using the MutableMapping function directly fails due to the private marker.
         # Using ordinary dict.pop would expose the internal structures.
         # So let's reinvent the wheel.
@@ -227,8 +233,10 @@ class HTTPHeaderDict(MutableMapping):
         with self.add instead of self.__setitem__
         """
         if len(args) > 1:
-            raise TypeError("extend() takes at most 1 positional "
-                            "arguments ({0} given)".format(len(args)))
+            raise TypeError(
+                "extend() takes at most 1 positional "
+                "arguments ({0} given)".format(len(args))
+            )
         other = args[0] if len(args) >= 1 else ()
 
         if isinstance(other, HTTPHeaderDict):
@@ -294,7 +302,7 @@ class HTTPHeaderDict(MutableMapping):
         """Iterate over all headers, merging duplicate ones together."""
         for key in self:
             val = self._container[key.lower()]
-            yield val[0], ', '.join(val[1:])
+            yield val[0], ", ".join(val[1:])
 
     def items(self):
         return list(self.iteritems())
@@ -305,15 +313,24 @@ class HTTPHeaderDict(MutableMapping):
         # python2.7 does not expose a proper API for exporting multiheaders
         # efficiently. This function re-reads raw lines from the message
         # object and extracts the multiheaders properly.
+        obs_fold_continued_leaders = (" ", "\t")
         headers = []
 
         for line in message.headers:
-            if line.startswith((' ', '\t')):
-                key, value = headers[-1]
-                headers[-1] = (key, value + '\r\n' + line.rstrip())
-                continue
+            if line.startswith(obs_fold_continued_leaders):
+                if not headers:
+                    # We received a header line that starts with OWS as described
+                    # in RFC-7230 S3.2.4. This indicates a multiline header, but
+                    # there exists no previous header to which we can attach it.
+                    raise InvalidHeader(
+                        "Header continuation with no previous header: %s" % line
+                    )
+                else:
+                    key, value = headers[-1]
+                    headers[-1] = (key, value + " " + line.strip())
+                    continue
 
-            key, value = line.split(':', 1)
+            key, value = line.split(":", 1)
             headers.append((key, value.strip()))
 
         return cls(headers)
diff --git a/lib/urllib3/connection.py b/lib/urllib3/connection.py
index c0d83299..3eeb1af5 100644
--- a/lib/urllib3/connection.py
+++ b/lib/urllib3/connection.py
@@ -2,7 +2,6 @@ from __future__ import absolute_import
 import datetime
 import logging
 import os
-import sys
 import socket
 from socket import error as SocketError, timeout as SocketTimeout
 import warnings
@@ -12,6 +11,7 @@ from .packages.six.moves.http_client import HTTPException  # noqa: F401
 
 try:  # Compiled with SSL?
     import ssl
+
     BaseSSLError = ssl.SSLError
 except (ImportError, AttributeError):  # Platform-specific: No SSL.
     ssl = None
@@ -20,10 +20,11 @@ except (ImportError, AttributeError):  # Platform-specific: No SSL.
         pass
 
 
-try:  # Python 3:
-    # Not a no-op, we're adding this to the namespace so it can be imported.
+try:
+    # Python 3: not a no-op, we're adding this to the namespace so it can be imported.
     ConnectionError = ConnectionError
-except NameError:  # Python 2:
+except NameError:
+    # Python 2
     class ConnectionError(Exception):
         pass
 
@@ -41,7 +42,7 @@ from .util.ssl_ import (
     resolve_ssl_version,
     assert_fingerprint,
     create_urllib3_context,
-    ssl_wrap_socket
+    ssl_wrap_socket,
 )
 
 
@@ -51,19 +52,16 @@ from ._collections import HTTPHeaderDict
 
 log = logging.getLogger(__name__)
 
-port_by_scheme = {
-    'http': 80,
-    'https': 443,
-}
+port_by_scheme = {"http": 80, "https": 443}
 
-# When updating RECENT_DATE, move it to
-# within two years of the current date, and no
-# earlier than 6 months ago.
-RECENT_DATE = datetime.date(2016, 1, 1)
+# When it comes time to update this value as a part of regular maintenance
+# (ie test_recent_date is failing) update it to ~6 months before the current date.
+RECENT_DATE = datetime.date(2019, 1, 1)
 
 
 class DummyConnection(object):
     """Used to detect a failed ConnectionCls import."""
+
     pass
 
 
@@ -77,9 +75,6 @@ class HTTPConnection(_HTTPConnection, object):
 
       - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
       - ``source_address``: Set the source address for the current connection.
-
-        .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x
-
       - ``socket_options``: Set specific options on the underlying socket. If not specified, then
         defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
         Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
@@ -94,7 +89,7 @@ class HTTPConnection(_HTTPConnection, object):
         Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
     """
 
-    default_port = port_by_scheme['http']
+    default_port = port_by_scheme["http"]
 
     #: Disable Nagle's algorithm by default.
     #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
@@ -104,26 +99,47 @@ class HTTPConnection(_HTTPConnection, object):
     is_verified = False
 
     def __init__(self, *args, **kw):
-        if six.PY3:  # Python 3
-            kw.pop('strict', None)
+        if not six.PY2:
+            kw.pop("strict", None)
 
-        # Pre-set source_address in case we have an older Python like 2.6.
-        self.source_address = kw.get('source_address')
-
-        if sys.version_info < (2, 7):  # Python 2.6
-            # _HTTPConnection on Python 2.6 will balk at this keyword arg, but
-            # not newer versions. We can still use it when creating a
-            # connection though, so we pop it *after* we have saved it as
-            # self.source_address.
-            kw.pop('source_address', None)
+        # Pre-set source_address.
+        self.source_address = kw.get("source_address")
 
         #: The socket options provided by the user. If no options are
         #: provided, we use the default options.
-        self.socket_options = kw.pop('socket_options', self.default_socket_options)
+        self.socket_options = kw.pop("socket_options", self.default_socket_options)
 
-        # Superclass also sets self.source_address in Python 2.7+.
         _HTTPConnection.__init__(self, *args, **kw)
 
+    @property
+    def host(self):
+        """
+        Getter method to remove any trailing dots that indicate the hostname is an FQDN.
+
+        In general, SSL certificates don't include the trailing dot indicating a
+        fully-qualified domain name, and thus, they don't validate properly when
+        checked against a domain name that includes the dot. In addition, some
+        servers may not expect to receive the trailing dot when provided.
+
+        However, the hostname with trailing dot is critical to DNS resolution; doing a
+        lookup with the trailing dot will properly only resolve the appropriate FQDN,
+        whereas a lookup without a trailing dot will search the system's search domain
+        list. Thus, it's important to keep the original host around for use only in
+        those cases where it's appropriate (i.e., when doing DNS lookup to establish the
+        actual TCP connection across which we're going to send HTTP requests).
+        """
+        return self._dns_host.rstrip(".")
+
+    @host.setter
+    def host(self, value):
+        """
+        Setter for the `host` property.
+
+        We assume that only urllib3 uses the _dns_host attribute; httplib itself
+        only uses `host`, and it seems reasonable that other libraries follow suit.
+        """
+        self._dns_host = value
+
     def _new_conn(self):
         """ Establish a socket connection and set nodelay settings on it.
 
@@ -131,32 +147,34 @@ class HTTPConnection(_HTTPConnection, object):
         """
         extra_kw = {}
         if self.source_address:
-            extra_kw['source_address'] = self.source_address
+            extra_kw["source_address"] = self.source_address
 
         if self.socket_options:
-            extra_kw['socket_options'] = self.socket_options
+            extra_kw["socket_options"] = self.socket_options
 
         try:
             conn = connection.create_connection(
-                (self.host, self.port), self.timeout, **extra_kw)
+                (self._dns_host, self.port), self.timeout, **extra_kw
+            )
 
-        except SocketTimeout as e:
+        except SocketTimeout:
             raise ConnectTimeoutError(
-                self, "Connection to %s timed out. (connect timeout=%s)" %
-                (self.host, self.timeout))
+                self,
+                "Connection to %s timed out. (connect timeout=%s)"
+                % (self.host, self.timeout),
+            )
 
         except SocketError as e:
             raise NewConnectionError(
-                self, "Failed to establish a new connection: %s" % e)
+                self, "Failed to establish a new connection: %s" % e
+            )
 
         return conn
 
     def _prepare_conn(self, conn):
         self.sock = conn
-        # the _tunnel_host attribute was added in python 2.6.3 (via
-        # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do
-        # not have them.
-        if getattr(self, '_tunnel_host', None):
+        # Google App Engine's httplib does not define _tunnel_host
+        if getattr(self, "_tunnel_host", None):
             # TODO: Fix tunnel so it doesn't depend on self.sock state.
             self._tunnel()
             # Mark this connection as not reusable
@@ -172,74 +190,99 @@ class HTTPConnection(_HTTPConnection, object):
         body with chunked encoding and not as one block
         """
         headers = HTTPHeaderDict(headers if headers is not None else {})
-        skip_accept_encoding = 'accept-encoding' in headers
-        skip_host = 'host' in headers
+        skip_accept_encoding = "accept-encoding" in headers
+        skip_host = "host" in headers
         self.putrequest(
-            method,
-            url,
-            skip_accept_encoding=skip_accept_encoding,
-            skip_host=skip_host
+            method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
         )
         for header, value in headers.items():
             self.putheader(header, value)
-        if 'transfer-encoding' not in headers:
-            self.putheader('Transfer-Encoding', 'chunked')
+        if "transfer-encoding" not in headers:
+            self.putheader("Transfer-Encoding", "chunked")
         self.endheaders()
 
         if body is not None:
-            stringish_types = six.string_types + (six.binary_type,)
+            stringish_types = six.string_types + (bytes,)
             if isinstance(body, stringish_types):
                 body = (body,)
             for chunk in body:
                 if not chunk:
                     continue
-                if not isinstance(chunk, six.binary_type):
-                    chunk = chunk.encode('utf8')
+                if not isinstance(chunk, bytes):
+                    chunk = chunk.encode("utf8")
                 len_str = hex(len(chunk))[2:]
-                self.send(len_str.encode('utf-8'))
-                self.send(b'\r\n')
+                self.send(len_str.encode("utf-8"))
+                self.send(b"\r\n")
                 self.send(chunk)
-                self.send(b'\r\n')
+                self.send(b"\r\n")
 
         # After the if clause, to always have a closed body
-        self.send(b'0\r\n\r\n')
+        self.send(b"0\r\n\r\n")
 
 
 class HTTPSConnection(HTTPConnection):
-    default_port = port_by_scheme['https']
+    default_port = port_by_scheme["https"]
 
     ssl_version = None
 
-    def __init__(self, host, port=None, key_file=None, cert_file=None,
-                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                 ssl_context=None, **kw):
+    def __init__(
+        self,
+        host,
+        port=None,
+        key_file=None,
+        cert_file=None,
+        key_password=None,
+        strict=None,
+        timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+        ssl_context=None,
+        server_hostname=None,
+        **kw
+    ):
 
-        HTTPConnection.__init__(self, host, port, strict=strict,
-                                timeout=timeout, **kw)
+        HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw)
 
         self.key_file = key_file
         self.cert_file = cert_file
+        self.key_password = key_password
         self.ssl_context = ssl_context
+        self.server_hostname = server_hostname
 
         # Required property for Google AppEngine 1.9.0 which otherwise causes
         # HTTPS requests to go out as HTTP. (See Issue #356)
-        self._protocol = 'https'
+        self._protocol = "https"
 
     def connect(self):
         conn = self._new_conn()
         self._prepare_conn(conn)
 
+        # Wrap socket using verification with the root certs in
+        # trusted_root_certs
+        default_ssl_context = False
         if self.ssl_context is None:
+            default_ssl_context = True
             self.ssl_context = create_urllib3_context(
-                ssl_version=resolve_ssl_version(None),
-                cert_reqs=resolve_cert_reqs(None),
+                ssl_version=resolve_ssl_version(self.ssl_version),
+                cert_reqs=resolve_cert_reqs(self.cert_reqs),
             )
 
+        # Try to load OS default certs if none are given.
+        # Works well on Windows (requires Python3.4+)
+        context = self.ssl_context
+        if (
+            not self.ca_certs
+            and not self.ca_cert_dir
+            and default_ssl_context
+            and hasattr(context, "load_default_certs")
+        ):
+            context.load_default_certs()
+
         self.sock = ssl_wrap_socket(
             sock=conn,
             keyfile=self.key_file,
             certfile=self.cert_file,
+            key_password=self.key_password,
             ssl_context=self.ssl_context,
+            server_hostname=self.server_hostname,
         )
 
 
@@ -248,32 +291,39 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     Based on httplib.HTTPSConnection but wraps the socket with
     SSL certification.
     """
+
     cert_reqs = None
     ca_certs = None
     ca_cert_dir = None
     ssl_version = None
     assert_fingerprint = None
 
-    def set_cert(self, key_file=None, cert_file=None,
-                 cert_reqs=None, ca_certs=None,
-                 assert_hostname=None, assert_fingerprint=None,
-                 ca_cert_dir=None):
+    def set_cert(
+        self,
+        key_file=None,
+        cert_file=None,
+        cert_reqs=None,
+        key_password=None,
+        ca_certs=None,
+        assert_hostname=None,
+        assert_fingerprint=None,
+        ca_cert_dir=None,
+    ):
         """
         This method should only be called once, before the connection is used.
         """
-        # If cert_reqs is not provided, we can try to guess. If the user gave
-        # us a cert database, we assume they want to use it: otherwise, if
-        # they gave us an SSL Context object we should use whatever is set for
-        # it.
+        # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
+        # have an SSLContext object in which case we'll use its verify_mode.
         if cert_reqs is None:
-            if ca_certs or ca_cert_dir:
-                cert_reqs = 'CERT_REQUIRED'
-            elif self.ssl_context is not None:
+            if self.ssl_context is not None:
                 cert_reqs = self.ssl_context.verify_mode
+            else:
+                cert_reqs = resolve_cert_reqs(None)
 
         self.key_file = key_file
         self.cert_file = cert_file
         self.cert_reqs = cert_reqs
+        self.key_password = key_password
         self.assert_hostname = assert_hostname
         self.assert_fingerprint = assert_fingerprint
         self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
@@ -282,12 +332,10 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     def connect(self):
         # Add certificate verification
         conn = self._new_conn()
-
         hostname = self.host
-        if getattr(self, '_tunnel_host', None):
-            # _tunnel_host was added in Python 2.6.3
-            # (See: http://hg.python.org/cpython/rev/0f57b30a152f)
 
+        # Google App Engine's httplib does not define _tunnel_host
+        if getattr(self, "_tunnel_host", None):
             self.sock = conn
             # Calls self._set_hostport(), so self.host is
             # self._tunnel_host below.
@@ -298,17 +346,25 @@ class VerifiedHTTPSConnection(HTTPSConnection):
             # Override the host with the one we're requesting data from.
             hostname = self._tunnel_host
 
+        server_hostname = hostname
+        if self.server_hostname is not None:
+            server_hostname = self.server_hostname
+
         is_time_off = datetime.date.today() < RECENT_DATE
         if is_time_off:
-            warnings.warn((
-                'System time is way off (before {0}). This will probably '
-                'lead to SSL verification errors').format(RECENT_DATE),
-                SystemTimeWarning
+            warnings.warn(
+                (
+                    "System time is way off (before {0}). This will probably "
+                    "lead to SSL verification errors"
+                ).format(RECENT_DATE),
+                SystemTimeWarning,
             )
 
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
+        default_ssl_context = False
         if self.ssl_context is None:
+            default_ssl_context = True
             self.ssl_context = create_urllib3_context(
                 ssl_version=resolve_ssl_version(self.ssl_version),
                 cert_reqs=resolve_cert_reqs(self.cert_reqs),
@@ -316,38 +372,56 @@ class VerifiedHTTPSConnection(HTTPSConnection):
 
         context = self.ssl_context
         context.verify_mode = resolve_cert_reqs(self.cert_reqs)
+
+        # Try to load OS default certs if none are given.
+        # Works well on Windows (requires Python3.4+)
+        if (
+            not self.ca_certs
+            and not self.ca_cert_dir
+            and default_ssl_context
+            and hasattr(context, "load_default_certs")
+        ):
+            context.load_default_certs()
+
         self.sock = ssl_wrap_socket(
             sock=conn,
             keyfile=self.key_file,
             certfile=self.cert_file,
+            key_password=self.key_password,
             ca_certs=self.ca_certs,
             ca_cert_dir=self.ca_cert_dir,
-            server_hostname=hostname,
-            ssl_context=context)
+            server_hostname=server_hostname,
+            ssl_context=context,
+        )
 
         if self.assert_fingerprint:
-            assert_fingerprint(self.sock.getpeercert(binary_form=True),
-                               self.assert_fingerprint)
-        elif context.verify_mode != ssl.CERT_NONE \
-                and not getattr(context, 'check_hostname', False) \
-                and self.assert_hostname is not False:
+            assert_fingerprint(
+                self.sock.getpeercert(binary_form=True), self.assert_fingerprint
+            )
+        elif (
+            context.verify_mode != ssl.CERT_NONE
+            and not getattr(context, "check_hostname", False)
+            and self.assert_hostname is not False
+        ):
             # While urllib3 attempts to always turn off hostname matching from
             # the TLS library, this cannot always be done. So we check whether
             # the TLS Library still thinks it's matching hostnames.
             cert = self.sock.getpeercert()
-            if not cert.get('subjectAltName', ()):
-                warnings.warn((
-                    'Certificate for {0} has no `subjectAltName`, falling back to check for a '
-                    '`commonName` for now. This feature is being removed by major browsers and '
-                    'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 '
-                    'for details.)'.format(hostname)),
-                    SubjectAltNameWarning
+            if not cert.get("subjectAltName", ()):
+                warnings.warn(
+                    (
+                        "Certificate for {0} has no `subjectAltName`, falling back to check for a "
+                        "`commonName` for now. This feature is being removed by major browsers and "
+                        "deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 "
+                        "for details.)".format(hostname)
+                    ),
+                    SubjectAltNameWarning,
                 )
-            _match_hostname(cert, self.assert_hostname or hostname)
+            _match_hostname(cert, self.assert_hostname or server_hostname)
 
         self.is_verified = (
-            context.verify_mode == ssl.CERT_REQUIRED or
-            self.assert_fingerprint is not None
+            context.verify_mode == ssl.CERT_REQUIRED
+            or self.assert_fingerprint is not None
         )
 
 
@@ -355,9 +429,10 @@ def _match_hostname(cert, asserted_hostname):
     try:
         match_hostname(cert, asserted_hostname)
     except CertificateError as e:
-        log.error(
-            'Certificate did not match expected hostname: %s. '
-            'Certificate: %s', asserted_hostname, cert
+        log.warning(
+            "Certificate did not match expected hostname: %s. " "Certificate: %s",
+            asserted_hostname,
+            cert,
         )
         # Add cert to exception and reraise so client code can inspect
         # the cert when catching the exception, if they want to
diff --git a/lib/urllib3/connectionpool.py b/lib/urllib3/connectionpool.py
index ec9600f8..e73fa57a 100644
--- a/lib/urllib3/connectionpool.py
+++ b/lib/urllib3/connectionpool.py
@@ -29,8 +29,11 @@ from .packages.six.moves import queue
 from .connection import (
     port_by_scheme,
     DummyConnection,
-    HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection,
-    HTTPException, BaseSSLError,
+    HTTPConnection,
+    HTTPSConnection,
+    VerifiedHTTPSConnection,
+    HTTPException,
+    BaseSSLError,
 )
 from .request import RequestMethods
 from .response import HTTPResponse
@@ -40,13 +43,16 @@ from .util.request import set_file_position
 from .util.response import assert_header_parsing
 from .util.retry import Retry
 from .util.timeout import Timeout
-from .util.url import get_host, Url
+from .util.url import (
+    get_host,
+    parse_url,
+    Url,
+    _normalize_host as normalize_host,
+    _encode_target,
+)
+from .util.queue import LifoQueue
 
 
-if six.PY2:
-    # Queue is imported for side effects on MS Windows
-    import Queue as _unused_module_Queue  # noqa: F401
-
 xrange = six.moves.xrange
 
 log = logging.getLogger(__name__)
@@ -62,19 +68,18 @@ class ConnectionPool(object):
     """
 
     scheme = None
-    QueueCls = queue.LifoQueue
+    QueueCls = LifoQueue
 
     def __init__(self, host, port=None):
         if not host:
             raise LocationValueError("No host specified.")
 
-        self.host = _ipv6_host(host).lower()
+        self.host = _normalize_host(host, scheme=self.scheme)
         self._proxy_host = host.lower()
         self.port = port
 
     def __str__(self):
-        return '%s(host=%r, port=%r)' % (type(self).__name__,
-                                         self.host, self.port)
+        return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)
 
     def __enter__(self):
         return self
@@ -92,7 +97,7 @@ class ConnectionPool(object):
 
 
 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
-_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK])
+_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
 
 
 class HTTPConnectionPool(ConnectionPool, RequestMethods):
@@ -155,15 +160,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         :class:`urllib3.connection.HTTPSConnection` instances.
     """
 
-    scheme = 'http'
+    scheme = "http"
     ConnectionCls = HTTPConnection
     ResponseCls = HTTPResponse
 
-    def __init__(self, host, port=None, strict=False,
-                 timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False,
-                 headers=None, retries=None,
-                 _proxy=None, _proxy_headers=None,
-                 **conn_kw):
+    def __init__(
+        self,
+        host,
+        port=None,
+        strict=False,
+        timeout=Timeout.DEFAULT_TIMEOUT,
+        maxsize=1,
+        block=False,
+        headers=None,
+        retries=None,
+        _proxy=None,
+        _proxy_headers=None,
+        **conn_kw
+    ):
         ConnectionPool.__init__(self, host, port)
         RequestMethods.__init__(self, headers)
 
@@ -197,19 +211,27 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
             # We cannot know if the user has added default socket options, so we cannot replace the
             # list.
-            self.conn_kw.setdefault('socket_options', [])
+            self.conn_kw.setdefault("socket_options", [])
 
     def _new_conn(self):
         """
         Return a fresh :class:`HTTPConnection`.
         """
         self.num_connections += 1
-        log.debug("Starting new HTTP connection (%d): %s",
-                  self.num_connections, self.host)
+        log.debug(
+            "Starting new HTTP connection (%d): %s:%s",
+            self.num_connections,
+            self.host,
+            self.port or "80",
+        )
 
-        conn = self.ConnectionCls(host=self.host, port=self.port,
-                                  timeout=self.timeout.connect_timeout,
-                                  strict=self.strict, **self.conn_kw)
+        conn = self.ConnectionCls(
+            host=self.host,
+            port=self.port,
+            timeout=self.timeout.connect_timeout,
+            strict=self.strict,
+            **self.conn_kw
+        )
         return conn
 
     def _get_conn(self, timeout=None):
@@ -233,16 +255,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         except queue.Empty:
             if self.block:
-                raise EmptyPoolError(self,
-                                     "Pool reached maximum size and no more "
-                                     "connections are allowed.")
+                raise EmptyPoolError(
+                    self,
+                    "Pool reached maximum size and no more " "connections are allowed.",
+                )
             pass  # Oh well, we'll create a new connection then
 
         # If this is a persistent connection, check if it got disconnected
         if conn and is_connection_dropped(conn):
             log.debug("Resetting dropped connection: %s", self.host)
             conn.close()
-            if getattr(conn, 'auto_open', 1) == 0:
+            if getattr(conn, "auto_open", 1) == 0:
                 # This is a proxied connection that has been mutated by
                 # httplib._tunnel() and cannot be reused (since it would
                 # attempt to bypass the proxy)
@@ -272,9 +295,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             pass
         except queue.Full:
             # This should never happen if self.block == True
-            log.warning(
-                "Connection pool is full, discarding connection: %s",
-                self.host)
+            log.warning("Connection pool is full, discarding connection: %s", self.host)
 
         # Connection never got put back into the pool, close it.
         if conn:
@@ -306,21 +327,30 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
 
         if isinstance(err, SocketTimeout):
-            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+            raise ReadTimeoutError(
+                self, url, "Read timed out. (read timeout=%s)" % timeout_value
+            )
 
         # See the above comment about EAGAIN in Python 3. In Python 2 we have
         # to specifically catch it and throw the timeout error
-        if hasattr(err, 'errno') and err.errno in _blocking_errnos:
-            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+        if hasattr(err, "errno") and err.errno in _blocking_errnos:
+            raise ReadTimeoutError(
+                self, url, "Read timed out. (read timeout=%s)" % timeout_value
+            )
 
         # Catch possible read timeouts thrown as SSL errors. If not the
         # case, rethrow the original. We need to do this because of:
         # http://bugs.python.org/issue10272
-        if 'timed out' in str(err) or 'did not complete (read)' in str(err):  # Python 2.6
-            raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
+        if "timed out" in str(err) or "did not complete (read)" in str(
+            err
+        ):  # Python < 2.7.4
+            raise ReadTimeoutError(
+                self, url, "Read timed out. (read timeout=%s)" % timeout_value
+            )
 
-    def _make_request(self, conn, method, url, timeout=_Default, chunked=False,
-                      **httplib_request_kw):
+    def _make_request(
+        self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
+    ):
         """
         Perform a request on a given urllib connection object taken from our
         pool.
@@ -360,7 +390,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         read_timeout = timeout_obj.read_timeout
 
         # App Engine doesn't have a sock attr
-        if getattr(conn, 'sock', None):
+        if getattr(conn, "sock", None):
             # In Python 3 socket.py will catch EAGAIN and return None when you
             # try and read into the file pointer created by http.client, which
             # instead raises a BadStatusLine exception. Instead of catching
@@ -368,7 +398,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # timeouts, check for a zero timeout before making the request.
             if read_timeout == 0:
                 raise ReadTimeoutError(
-                    self, url, "Read timed out. (read timeout=%s)" % read_timeout)
+                    self, url, "Read timed out. (read timeout=%s)" % read_timeout
+                )
             if read_timeout is Timeout.DEFAULT_TIMEOUT:
                 conn.sock.settimeout(socket.getdefaulttimeout())
             else:  # None or a value
@@ -376,31 +407,45 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         # Receive the response from the server
         try:
-            try:  # Python 2.7, use buffering of HTTP responses
+            try:
+                # Python 2.7, use buffering of HTTP responses
                 httplib_response = conn.getresponse(buffering=True)
-            except TypeError:  # Python 2.6 and older, Python 3
+            except TypeError:
+                # Python 3
                 try:
                     httplib_response = conn.getresponse()
-                except Exception as e:
-                    # Remove the TypeError from the exception chain in Python 3;
-                    # otherwise it looks like a programming error was the cause.
+                except BaseException as e:
+                    # Remove the TypeError from the exception chain in
+                    # Python 3 (including for exceptions like SystemExit).
+                    # Otherwise it looks like a bug in the code.
                     six.raise_from(e, None)
         except (SocketTimeout, BaseSSLError, SocketError) as e:
             self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
             raise
 
         # AppEngine doesn't have a version attr.
-        http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
-        log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port,
-                  method, url, http_version, httplib_response.status,
-                  httplib_response.length)
+        http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
+        log.debug(
+            '%s://%s:%s "%s %s %s" %s %s',
+            self.scheme,
+            self.host,
+            self.port,
+            method,
+            url,
+            http_version,
+            httplib_response.status,
+            httplib_response.length,
+        )
 
         try:
             assert_header_parsing(httplib_response.msg)
         except (HeaderParsingError, TypeError) as hpe:  # Platform-specific: Python 3
             log.warning(
-                'Failed to parse headers (url=%s): %s',
-                self._absolute_url(url), hpe, exc_info=True)
+                "Failed to parse headers (url=%s): %s",
+                self._absolute_url(url),
+                hpe,
+                exc_info=True,
+            )
 
         return httplib_response
 
@@ -411,6 +456,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         """
         Close all pooled connections and disable the pool.
         """
+        if self.pool is None:
+            return
         # Disable access to the pool
         old_pool, self.pool = self.pool, None
 
@@ -428,13 +475,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         Check if the given ``url`` is a member of the same host as this
         connection pool.
         """
-        if url.startswith('/'):
+        if url.startswith("/"):
             return True
 
         # TODO: Add optional support for socket.gethostbyname checking.
         scheme, host, port = get_host(url)
-
-        host = _ipv6_host(host).lower()
+        if host is not None:
+            host = _normalize_host(host, scheme=scheme)
 
         # Use explicit default port for comparison when none is given
         if self.port and not port:
@@ -444,10 +491,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         return (scheme, host, port) == (self.scheme, self.host, self.port)
 
-    def urlopen(self, method, url, body=None, headers=None, retries=None,
-                redirect=True, assert_same_host=True, timeout=_Default,
-                pool_timeout=None, release_conn=None, chunked=False,
-                body_pos=None, **response_kw):
+    def urlopen(
+        self,
+        method,
+        url,
+        body=None,
+        headers=None,
+        retries=None,
+        redirect=True,
+        assert_same_host=True,
+        timeout=_Default,
+        pool_timeout=None,
+        release_conn=None,
+        chunked=False,
+        body_pos=None,
+        **response_kw
+    ):
         """
         Get a connection from the pool and perform an HTTP request. This is the
         lowest level call for making a request, so you'll need to specify all
@@ -545,12 +604,18 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
 
         if release_conn is None:
-            release_conn = response_kw.get('preload_content', True)
+            release_conn = response_kw.get("preload_content", True)
 
         # Check host
         if assert_same_host and not self.is_same_host(url):
             raise HostChangedError(self, url, retries)
 
+        # Ensure that the URL we're connecting to is properly encoded
+        if url.startswith("/"):
+            url = six.ensure_str(_encode_target(url))
+        else:
+            url = six.ensure_str(parse_url(url).url)
+
         conn = None
 
         # Track whether `conn` needs to be released before
@@ -567,7 +632,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         # Merge the proxy headers. Only do this in HTTP. We have to copy the
         # headers dict so we can safely change it without those changes being
         # reflected in anyone else's copy.
-        if self.scheme == 'http':
+        if self.scheme == "http":
             headers = headers.copy()
             headers.update(self.proxy_headers)
 
@@ -590,15 +655,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
             conn.timeout = timeout_obj.connect_timeout
 
-            is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None)
+            is_new_proxy_conn = self.proxy is not None and not getattr(
+                conn, "sock", None
+            )
             if is_new_proxy_conn:
                 self._prepare_proxy(conn)
 
             # Make the request on the httplib connection object.
-            httplib_response = self._make_request(conn, method, url,
-                                                  timeout=timeout_obj,
-                                                  body=body, headers=headers,
-                                                  chunked=chunked)
+            httplib_response = self._make_request(
+                conn,
+                method,
+                url,
+                timeout=timeout_obj,
+                body=body,
+                headers=headers,
+                chunked=chunked,
+            )
 
             # If we're going to release the connection in ``finally:``, then
             # the response doesn't need to know about the connection. Otherwise
@@ -607,14 +679,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             response_conn = conn if not release_conn else None
 
             # Pass method to Response for length checking
-            response_kw['request_method'] = method
+            response_kw["request_method"] = method
 
             # Import httplib's response into our own wrapper object
-            response = self.ResponseCls.from_httplib(httplib_response,
-                                                     pool=self,
-                                                     connection=response_conn,
-                                                     retries=retries,
-                                                     **response_kw)
+            response = self.ResponseCls.from_httplib(
+                httplib_response,
+                pool=self,
+                connection=response_conn,
+                retries=retries,
+                **response_kw
+            )
 
             # Everything went great!
             clean_exit = True
@@ -623,20 +697,28 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Timed out by queue.
             raise EmptyPoolError(self, "No pool connections are available.")
 
-        except (TimeoutError, HTTPException, SocketError, ProtocolError,
-                BaseSSLError, SSLError, CertificateError) as e:
+        except (
+            TimeoutError,
+            HTTPException,
+            SocketError,
+            ProtocolError,
+            BaseSSLError,
+            SSLError,
+            CertificateError,
+        ) as e:
             # Discard the connection for these exceptions. It will be
             # replaced during the next _get_conn() call.
             clean_exit = False
             if isinstance(e, (BaseSSLError, CertificateError)):
                 e = SSLError(e)
             elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
-                e = ProxyError('Cannot connect to proxy.', e)
+                e = ProxyError("Cannot connect to proxy.", e)
             elif isinstance(e, (SocketError, HTTPException)):
-                e = ProtocolError('Connection aborted.', e)
+                e = ProtocolError("Connection aborted.", e)
 
-            retries = retries.increment(method, url, error=e, _pool=self,
-                                        _stacktrace=sys.exc_info()[2])
+            retries = retries.increment(
+                method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
+            )
             retries.sleep()
 
             # Keep track of the error for the retry warning.
@@ -659,28 +741,47 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         if not conn:
             # Try again
-            log.warning("Retrying (%r) after connection "
-                        "broken by '%r': %s", retries, err, url)
-            return self.urlopen(method, url, body, headers, retries,
-                                redirect, assert_same_host,
-                                timeout=timeout, pool_timeout=pool_timeout,
-                                release_conn=release_conn, body_pos=body_pos,
-                                **response_kw)
+            log.warning(
+                "Retrying (%r) after connection " "broken by '%r': %s",
+                retries,
+                err,
+                url,
+            )
+            return self.urlopen(
+                method,
+                url,
+                body,
+                headers,
+                retries,
+                redirect,
+                assert_same_host,
+                timeout=timeout,
+                pool_timeout=pool_timeout,
+                release_conn=release_conn,
+                body_pos=body_pos,
+                **response_kw
+            )
 
         def drain_and_release_conn(response):
             try:
                 # discard any remaining response body, the connection will be
                 # released back to the pool once the entire response is read
                 response.read()
-            except (TimeoutError, HTTPException, SocketError, ProtocolError,
-                    BaseSSLError, SSLError) as e:
+            except (
+                TimeoutError,
+                HTTPException,
+                SocketError,
+                ProtocolError,
+                BaseSSLError,
+                SSLError,
+            ):
                 pass
 
         # Handle redirect?
         redirect_location = redirect and response.get_redirect_location()
         if redirect_location:
             if response.status == 303:
-                method = 'GET'
+                method = "GET"
 
             try:
                 retries = retries.increment(method, url, response=response, _pool=self)
@@ -698,15 +799,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             retries.sleep_for_retry(response)
             log.debug("Redirecting %s -> %s", url, redirect_location)
             return self.urlopen(
-                method, redirect_location, body, headers,
-                retries=retries, redirect=redirect,
+                method,
+                redirect_location,
+                body,
+                headers,
+                retries=retries,
+                redirect=redirect,
                 assert_same_host=assert_same_host,
-                timeout=timeout, pool_timeout=pool_timeout,
-                release_conn=release_conn, body_pos=body_pos,
-                **response_kw)
+                timeout=timeout,
+                pool_timeout=pool_timeout,
+                release_conn=release_conn,
+                body_pos=body_pos,
+                **response_kw
+            )
 
         # Check if we should retry the HTTP response.
-        has_retry_after = bool(response.getheader('Retry-After'))
+        has_retry_after = bool(response.getheader("Retry-After"))
         if retries.is_retry(method, response.status, has_retry_after):
             try:
                 retries = retries.increment(method, url, response=response, _pool=self)
@@ -724,12 +832,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             retries.sleep(response)
             log.debug("Retry: %s", url)
             return self.urlopen(
-                method, url, body, headers,
-                retries=retries, redirect=redirect,
+                method,
+                url,
+                body,
+                headers,
+                retries=retries,
+                redirect=redirect,
                 assert_same_host=assert_same_host,
-                timeout=timeout, pool_timeout=pool_timeout,
+                timeout=timeout,
+                pool_timeout=pool_timeout,
                 release_conn=release_conn,
-                body_pos=body_pos, **response_kw)
+                body_pos=body_pos,
+                **response_kw
+            )
 
         return response
 
@@ -747,33 +862,57 @@ class HTTPSConnectionPool(HTTPConnectionPool):
     If ``assert_hostname`` is False, no verification is done.
 
     The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
-    ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is
-    available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
+    ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
+    is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
     the connection socket into an SSL socket.
     """
 
-    scheme = 'https'
+    scheme = "https"
     ConnectionCls = HTTPSConnection
 
-    def __init__(self, host, port=None,
-                 strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1,
-                 block=False, headers=None, retries=None,
-                 _proxy=None, _proxy_headers=None,
-                 key_file=None, cert_file=None, cert_reqs=None,
-                 ca_certs=None, ssl_version=None,
-                 assert_hostname=None, assert_fingerprint=None,
-                 ca_cert_dir=None, **conn_kw):
+    def __init__(
+        self,
+        host,
+        port=None,
+        strict=False,
+        timeout=Timeout.DEFAULT_TIMEOUT,
+        maxsize=1,
+        block=False,
+        headers=None,
+        retries=None,
+        _proxy=None,
+        _proxy_headers=None,
+        key_file=None,
+        cert_file=None,
+        cert_reqs=None,
+        key_password=None,
+        ca_certs=None,
+        ssl_version=None,
+        assert_hostname=None,
+        assert_fingerprint=None,
+        ca_cert_dir=None,
+        **conn_kw
+    ):
 
-        HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize,
-                                    block, headers, retries, _proxy, _proxy_headers,
-                                    **conn_kw)
-
-        if ca_certs and cert_reqs is None:
-            cert_reqs = 'CERT_REQUIRED'
+        HTTPConnectionPool.__init__(
+            self,
+            host,
+            port,
+            strict,
+            timeout,
+            maxsize,
+            block,
+            headers,
+            retries,
+            _proxy,
+            _proxy_headers,
+            **conn_kw
+        )
 
         self.key_file = key_file
         self.cert_file = cert_file
         self.cert_reqs = cert_reqs
+        self.key_password = key_password
         self.ca_certs = ca_certs
         self.ca_cert_dir = ca_cert_dir
         self.ssl_version = ssl_version
@@ -787,13 +926,16 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         """
 
         if isinstance(conn, VerifiedHTTPSConnection):
-            conn.set_cert(key_file=self.key_file,
-                          cert_file=self.cert_file,
-                          cert_reqs=self.cert_reqs,
-                          ca_certs=self.ca_certs,
-                          ca_cert_dir=self.ca_cert_dir,
-                          assert_hostname=self.assert_hostname,
-                          assert_fingerprint=self.assert_fingerprint)
+            conn.set_cert(
+                key_file=self.key_file,
+                key_password=self.key_password,
+                cert_file=self.cert_file,
+                cert_reqs=self.cert_reqs,
+                ca_certs=self.ca_certs,
+                ca_cert_dir=self.ca_cert_dir,
+                assert_hostname=self.assert_hostname,
+                assert_fingerprint=self.assert_fingerprint,
+            )
             conn.ssl_version = self.ssl_version
         return conn
 
@@ -802,17 +944,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         Establish tunnel connection early, because otherwise httplib
         would improperly set Host: header to proxy's IP:port.
         """
-        # Python 2.7+
-        try:
-            set_tunnel = conn.set_tunnel
-        except AttributeError:  # Platform-specific: Python 2.6
-            set_tunnel = conn._set_tunnel
-
-        if sys.version_info <= (2, 6, 4) and not self.proxy_headers:  # Python 2.6.4 and older
-            set_tunnel(self._proxy_host, self.port)
-        else:
-            set_tunnel(self._proxy_host, self.port, self.proxy_headers)
-
+        conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
         conn.connect()
 
     def _new_conn(self):
@@ -820,12 +952,17 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         Return a fresh :class:`httplib.HTTPSConnection`.
         """
         self.num_connections += 1
-        log.debug("Starting new HTTPS connection (%d): %s",
-                  self.num_connections, self.host)
+        log.debug(
+            "Starting new HTTPS connection (%d): %s:%s",
+            self.num_connections,
+            self.host,
+            self.port or "443",
+        )
 
         if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
-            raise SSLError("Can't connect to HTTPS URL because the SSL "
-                           "module is not available.")
+            raise SSLError(
+                "Can't connect to HTTPS URL because the SSL " "module is not available."
+            )
 
         actual_host = self.host
         actual_port = self.port
@@ -833,9 +970,16 @@ class HTTPSConnectionPool(HTTPConnectionPool):
             actual_host = self.proxy.host
             actual_port = self.proxy.port
 
-        conn = self.ConnectionCls(host=actual_host, port=actual_port,
-                                  timeout=self.timeout.connect_timeout,
-                                  strict=self.strict, **self.conn_kw)
+        conn = self.ConnectionCls(
+            host=actual_host,
+            port=actual_port,
+            timeout=self.timeout.connect_timeout,
+            strict=self.strict,
+            cert_file=self.cert_file,
+            key_file=self.key_file,
+            key_password=self.key_password,
+            **self.conn_kw
+        )
 
         return self._prepare_conn(conn)
 
@@ -846,16 +990,19 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         super(HTTPSConnectionPool, self)._validate_conn(conn)
 
         # Force connect early to allow us to validate the connection.
-        if not getattr(conn, 'sock', None):  # AppEngine might not have  `.sock`
+        if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
             conn.connect()
 
         if not conn.is_verified:
-            warnings.warn((
-                'Unverified HTTPS request is being made. '
-                'Adding certificate verification is strongly advised. See: '
-                'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
-                '#ssl-warnings'),
-                InsecureRequestWarning)
+            warnings.warn(
+                (
+                    "Unverified HTTPS request is being made. "
+                    "Adding certificate verification is strongly advised. See: "
+                    "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
+                    "#ssl-warnings"
+                ),
+                InsecureRequestWarning,
+            )
 
 
 def connection_from_url(url, **kw):
@@ -880,26 +1027,25 @@ def connection_from_url(url, **kw):
     """
     scheme, host, port = get_host(url)
     port = port or port_by_scheme.get(scheme, 80)
-    if scheme == 'https':
+    if scheme == "https":
         return HTTPSConnectionPool(host, port=port, **kw)
     else:
         return HTTPConnectionPool(host, port=port, **kw)
 
 
-def _ipv6_host(host):
+def _normalize_host(host, scheme):
     """
-    Process IPv6 address literals
+    Normalize hosts for comparisons and use with sockets.
     """
 
+    host = normalize_host(host, scheme)
+
     # httplib doesn't like it when we include brackets in IPv6 addresses
     # Specifically, if we include brackets but also pass the port then
     # httplib crazily doubles up the square brackets on the Host header.
     # Instead, we need to make sure we never pass ``None`` as the port.
     # However, for backward compatibility reasons we can't actually
     # *assert* that.  See http://bugs.python.org/issue28539
-    #
-    # Also if an IPv6 address literal has a zone identifier, the
-    # percent sign might be URIencoded, convert it back into ASCII
-    if host.startswith('[') and host.endswith(']'):
-        host = host.replace('%25', '%').strip('[]')
+    if host.startswith("[") and host.endswith("]"):
+        host = host[1:-1]
     return host
diff --git a/lib/tornado/platform/__init__.py b/lib/urllib3/contrib/__init__.py
old mode 100755
new mode 100644
similarity index 100%
rename from lib/tornado/platform/__init__.py
rename to lib/urllib3/contrib/__init__.py
diff --git a/lib/urllib3/contrib/_appengine_environ.py b/lib/urllib3/contrib/_appengine_environ.py
new file mode 100644
index 00000000..c909010b
--- /dev/null
+++ b/lib/urllib3/contrib/_appengine_environ.py
@@ -0,0 +1,32 @@
+"""
+This module provides means to detect the App Engine environment.
+"""
+
+import os
+
+
+def is_appengine():
+    return is_local_appengine() or is_prod_appengine() or is_prod_appengine_mvms()
+
+
+def is_appengine_sandbox():
+    return is_appengine() and not is_prod_appengine_mvms()
+
+
+def is_local_appengine():
+    return (
+        "APPENGINE_RUNTIME" in os.environ
+        and "Development/" in os.environ["SERVER_SOFTWARE"]
+    )
+
+
+def is_prod_appengine():
+    return (
+        "APPENGINE_RUNTIME" in os.environ
+        and "Google App Engine/" in os.environ["SERVER_SOFTWARE"]
+        and not is_prod_appengine_mvms()
+    )
+
+
+def is_prod_appengine_mvms():
+    return os.environ.get("GAE_VM", False) == "true"
diff --git a/lib/tornado/test/__init__.py b/lib/urllib3/contrib/_securetransport/__init__.py
old mode 100755
new mode 100644
similarity index 100%
rename from lib/tornado/test/__init__.py
rename to lib/urllib3/contrib/_securetransport/__init__.py
diff --git a/lib/urllib3/contrib/_securetransport/bindings.py b/lib/urllib3/contrib/_securetransport/bindings.py
index bcf41c02..b46e1e3b 100644
--- a/lib/urllib3/contrib/_securetransport/bindings.py
+++ b/lib/urllib3/contrib/_securetransport/bindings.py
@@ -34,29 +34,35 @@ from __future__ import absolute_import
 import platform
 from ctypes.util import find_library
 from ctypes import (
-    c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long,
-    c_bool
+    c_void_p,
+    c_int32,
+    c_char_p,
+    c_size_t,
+    c_byte,
+    c_uint32,
+    c_ulong,
+    c_long,
+    c_bool,
 )
 from ctypes import CDLL, POINTER, CFUNCTYPE
 
 
-security_path = find_library('Security')
+security_path = find_library("Security")
 if not security_path:
-    raise ImportError('The library Security could not be found')
+    raise ImportError("The library Security could not be found")
 
 
-core_foundation_path = find_library('CoreFoundation')
+core_foundation_path = find_library("CoreFoundation")
 if not core_foundation_path:
-    raise ImportError('The library CoreFoundation could not be found')
+    raise ImportError("The library CoreFoundation could not be found")
 
 
 version = platform.mac_ver()[0]
-version_info = tuple(map(int, version.split('.')))
+version_info = tuple(map(int, version.split(".")))
 if version_info < (10, 8):
     raise OSError(
-        'Only OS X 10.8 and newer are supported, not %s.%s' % (
-            version_info[0], version_info[1]
-        )
+        "Only OS X 10.8 and newer are supported, not %s.%s"
+        % (version_info[0], version_info[1])
     )
 
 Security = CDLL(security_path, use_errno=True)
@@ -129,27 +135,19 @@ try:
     Security.SecKeyGetTypeID.argtypes = []
     Security.SecKeyGetTypeID.restype = CFTypeID
 
-    Security.SecCertificateCreateWithData.argtypes = [
-        CFAllocatorRef,
-        CFDataRef
-    ]
+    Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef]
     Security.SecCertificateCreateWithData.restype = SecCertificateRef
 
-    Security.SecCertificateCopyData.argtypes = [
-        SecCertificateRef
-    ]
+    Security.SecCertificateCopyData.argtypes = [SecCertificateRef]
     Security.SecCertificateCopyData.restype = CFDataRef
 
-    Security.SecCopyErrorMessageString.argtypes = [
-        OSStatus,
-        c_void_p
-    ]
+    Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
     Security.SecCopyErrorMessageString.restype = CFStringRef
 
     Security.SecIdentityCreateWithCertificate.argtypes = [
         CFTypeRef,
         SecCertificateRef,
-        POINTER(SecIdentityRef)
+        POINTER(SecIdentityRef),
     ]
     Security.SecIdentityCreateWithCertificate.restype = OSStatus
 
@@ -159,201 +157,126 @@ try:
         c_void_p,
         Boolean,
         c_void_p,
-        POINTER(SecKeychainRef)
+        POINTER(SecKeychainRef),
     ]
     Security.SecKeychainCreate.restype = OSStatus
 
-    Security.SecKeychainDelete.argtypes = [
-        SecKeychainRef
-    ]
+    Security.SecKeychainDelete.argtypes = [SecKeychainRef]
     Security.SecKeychainDelete.restype = OSStatus
 
     Security.SecPKCS12Import.argtypes = [
         CFDataRef,
         CFDictionaryRef,
-        POINTER(CFArrayRef)
+        POINTER(CFArrayRef),
     ]
     Security.SecPKCS12Import.restype = OSStatus
 
     SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t))
-    SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t))
+    SSLWriteFunc = CFUNCTYPE(
+        OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)
+    )
 
-    Security.SSLSetIOFuncs.argtypes = [
-        SSLContextRef,
-        SSLReadFunc,
-        SSLWriteFunc
-    ]
+    Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc]
     Security.SSLSetIOFuncs.restype = OSStatus
 
-    Security.SSLSetPeerID.argtypes = [
-        SSLContextRef,
-        c_char_p,
-        c_size_t
-    ]
+    Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t]
     Security.SSLSetPeerID.restype = OSStatus
 
-    Security.SSLSetCertificate.argtypes = [
-        SSLContextRef,
-        CFArrayRef
-    ]
+    Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef]
     Security.SSLSetCertificate.restype = OSStatus
 
-    Security.SSLSetCertificateAuthorities.argtypes = [
-        SSLContextRef,
-        CFTypeRef,
-        Boolean
-    ]
+    Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean]
     Security.SSLSetCertificateAuthorities.restype = OSStatus
 
-    Security.SSLSetConnection.argtypes = [
-        SSLContextRef,
-        SSLConnectionRef
-    ]
+    Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef]
     Security.SSLSetConnection.restype = OSStatus
 
-    Security.SSLSetPeerDomainName.argtypes = [
-        SSLContextRef,
-        c_char_p,
-        c_size_t
-    ]
+    Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t]
     Security.SSLSetPeerDomainName.restype = OSStatus
 
-    Security.SSLHandshake.argtypes = [
-        SSLContextRef
-    ]
+    Security.SSLHandshake.argtypes = [SSLContextRef]
     Security.SSLHandshake.restype = OSStatus
 
-    Security.SSLRead.argtypes = [
-        SSLContextRef,
-        c_char_p,
-        c_size_t,
-        POINTER(c_size_t)
-    ]
+    Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)]
     Security.SSLRead.restype = OSStatus
 
-    Security.SSLWrite.argtypes = [
-        SSLContextRef,
-        c_char_p,
-        c_size_t,
-        POINTER(c_size_t)
-    ]
+    Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)]
     Security.SSLWrite.restype = OSStatus
 
-    Security.SSLClose.argtypes = [
-        SSLContextRef
-    ]
+    Security.SSLClose.argtypes = [SSLContextRef]
     Security.SSLClose.restype = OSStatus
 
-    Security.SSLGetNumberSupportedCiphers.argtypes = [
-        SSLContextRef,
-        POINTER(c_size_t)
-    ]
+    Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)]
     Security.SSLGetNumberSupportedCiphers.restype = OSStatus
 
     Security.SSLGetSupportedCiphers.argtypes = [
         SSLContextRef,
         POINTER(SSLCipherSuite),
-        POINTER(c_size_t)
+        POINTER(c_size_t),
     ]
     Security.SSLGetSupportedCiphers.restype = OSStatus
 
     Security.SSLSetEnabledCiphers.argtypes = [
         SSLContextRef,
         POINTER(SSLCipherSuite),
-        c_size_t
+        c_size_t,
     ]
     Security.SSLSetEnabledCiphers.restype = OSStatus
 
-    Security.SSLGetNumberEnabledCiphers.argtype = [
-        SSLContextRef,
-        POINTER(c_size_t)
-    ]
+    Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)]
     Security.SSLGetNumberEnabledCiphers.restype = OSStatus
 
     Security.SSLGetEnabledCiphers.argtypes = [
         SSLContextRef,
         POINTER(SSLCipherSuite),
-        POINTER(c_size_t)
+        POINTER(c_size_t),
     ]
     Security.SSLGetEnabledCiphers.restype = OSStatus
 
-    Security.SSLGetNegotiatedCipher.argtypes = [
-        SSLContextRef,
-        POINTER(SSLCipherSuite)
-    ]
+    Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)]
     Security.SSLGetNegotiatedCipher.restype = OSStatus
 
     Security.SSLGetNegotiatedProtocolVersion.argtypes = [
         SSLContextRef,
-        POINTER(SSLProtocol)
+        POINTER(SSLProtocol),
     ]
     Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus
 
-    Security.SSLCopyPeerTrust.argtypes = [
-        SSLContextRef,
-        POINTER(SecTrustRef)
-    ]
+    Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)]
     Security.SSLCopyPeerTrust.restype = OSStatus
 
-    Security.SecTrustSetAnchorCertificates.argtypes = [
-        SecTrustRef,
-        CFArrayRef
-    ]
+    Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef]
     Security.SecTrustSetAnchorCertificates.restype = OSStatus
 
-    Security.SecTrustSetAnchorCertificatesOnly.argstypes = [
-        SecTrustRef,
-        Boolean
-    ]
+    Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean]
     Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus
 
-    Security.SecTrustEvaluate.argtypes = [
-        SecTrustRef,
-        POINTER(SecTrustResultType)
-    ]
+    Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)]
     Security.SecTrustEvaluate.restype = OSStatus
 
-    Security.SecTrustGetCertificateCount.argtypes = [
-        SecTrustRef
-    ]
+    Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef]
     Security.SecTrustGetCertificateCount.restype = CFIndex
 
-    Security.SecTrustGetCertificateAtIndex.argtypes = [
-        SecTrustRef,
-        CFIndex
-    ]
+    Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex]
     Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef
 
     Security.SSLCreateContext.argtypes = [
         CFAllocatorRef,
         SSLProtocolSide,
-        SSLConnectionType
+        SSLConnectionType,
     ]
     Security.SSLCreateContext.restype = SSLContextRef
 
-    Security.SSLSetSessionOption.argtypes = [
-        SSLContextRef,
-        SSLSessionOption,
-        Boolean
-    ]
+    Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean]
     Security.SSLSetSessionOption.restype = OSStatus
 
-    Security.SSLSetProtocolVersionMin.argtypes = [
-        SSLContextRef,
-        SSLProtocol
-    ]
+    Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol]
     Security.SSLSetProtocolVersionMin.restype = OSStatus
 
-    Security.SSLSetProtocolVersionMax.argtypes = [
-        SSLContextRef,
-        SSLProtocol
-    ]
+    Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol]
     Security.SSLSetProtocolVersionMax.restype = OSStatus
 
-    Security.SecCopyErrorMessageString.argtypes = [
-        OSStatus,
-        c_void_p
-    ]
+    Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
     Security.SecCopyErrorMessageString.restype = CFStringRef
 
     Security.SSLReadFunc = SSLReadFunc
@@ -369,64 +292,47 @@ try:
     Security.OSStatus = OSStatus
 
     Security.kSecImportExportPassphrase = CFStringRef.in_dll(
-        Security, 'kSecImportExportPassphrase'
+        Security, "kSecImportExportPassphrase"
     )
     Security.kSecImportItemIdentity = CFStringRef.in_dll(
-        Security, 'kSecImportItemIdentity'
+        Security, "kSecImportItemIdentity"
     )
 
     # CoreFoundation time!
-    CoreFoundation.CFRetain.argtypes = [
-        CFTypeRef
-    ]
+    CoreFoundation.CFRetain.argtypes = [CFTypeRef]
     CoreFoundation.CFRetain.restype = CFTypeRef
 
-    CoreFoundation.CFRelease.argtypes = [
-        CFTypeRef
-    ]
+    CoreFoundation.CFRelease.argtypes = [CFTypeRef]
     CoreFoundation.CFRelease.restype = None
 
-    CoreFoundation.CFGetTypeID.argtypes = [
-        CFTypeRef
-    ]
+    CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef]
     CoreFoundation.CFGetTypeID.restype = CFTypeID
 
     CoreFoundation.CFStringCreateWithCString.argtypes = [
         CFAllocatorRef,
         c_char_p,
-        CFStringEncoding
+        CFStringEncoding,
     ]
     CoreFoundation.CFStringCreateWithCString.restype = CFStringRef
 
-    CoreFoundation.CFStringGetCStringPtr.argtypes = [
-        CFStringRef,
-        CFStringEncoding
-    ]
+    CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding]
     CoreFoundation.CFStringGetCStringPtr.restype = c_char_p
 
     CoreFoundation.CFStringGetCString.argtypes = [
         CFStringRef,
         c_char_p,
         CFIndex,
-        CFStringEncoding
+        CFStringEncoding,
     ]
     CoreFoundation.CFStringGetCString.restype = c_bool
 
-    CoreFoundation.CFDataCreate.argtypes = [
-        CFAllocatorRef,
-        c_char_p,
-        CFIndex
-    ]
+    CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex]
     CoreFoundation.CFDataCreate.restype = CFDataRef
 
-    CoreFoundation.CFDataGetLength.argtypes = [
-        CFDataRef
-    ]
+    CoreFoundation.CFDataGetLength.argtypes = [CFDataRef]
     CoreFoundation.CFDataGetLength.restype = CFIndex
 
-    CoreFoundation.CFDataGetBytePtr.argtypes = [
-        CFDataRef
-    ]
+    CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef]
     CoreFoundation.CFDataGetBytePtr.restype = c_void_p
 
     CoreFoundation.CFDictionaryCreate.argtypes = [
@@ -435,14 +341,11 @@ try:
         POINTER(CFTypeRef),
         CFIndex,
         CFDictionaryKeyCallBacks,
-        CFDictionaryValueCallBacks
+        CFDictionaryValueCallBacks,
     ]
     CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef
 
-    CoreFoundation.CFDictionaryGetValue.argtypes = [
-        CFDictionaryRef,
-        CFTypeRef
-    ]
+    CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef]
     CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef
 
     CoreFoundation.CFArrayCreate.argtypes = [
@@ -456,36 +359,30 @@ try:
     CoreFoundation.CFArrayCreateMutable.argtypes = [
         CFAllocatorRef,
         CFIndex,
-        CFArrayCallBacks
+        CFArrayCallBacks,
     ]
     CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef
 
-    CoreFoundation.CFArrayAppendValue.argtypes = [
-        CFMutableArrayRef,
-        c_void_p
-    ]
+    CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p]
     CoreFoundation.CFArrayAppendValue.restype = None
 
-    CoreFoundation.CFArrayGetCount.argtypes = [
-        CFArrayRef
-    ]
+    CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef]
     CoreFoundation.CFArrayGetCount.restype = CFIndex
 
-    CoreFoundation.CFArrayGetValueAtIndex.argtypes = [
-        CFArrayRef,
-        CFIndex
-    ]
+    CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex]
     CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p
 
     CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll(
-        CoreFoundation, 'kCFAllocatorDefault'
+        CoreFoundation, "kCFAllocatorDefault"
+    )
+    CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(
+        CoreFoundation, "kCFTypeArrayCallBacks"
     )
-    CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks')
     CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll(
-        CoreFoundation, 'kCFTypeDictionaryKeyCallBacks'
+        CoreFoundation, "kCFTypeDictionaryKeyCallBacks"
     )
     CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll(
-        CoreFoundation, 'kCFTypeDictionaryValueCallBacks'
+        CoreFoundation, "kCFTypeDictionaryValueCallBacks"
     )
 
     CoreFoundation.CFTypeRef = CFTypeRef
@@ -494,7 +391,7 @@ try:
     CoreFoundation.CFDictionaryRef = CFDictionaryRef
 
 except (AttributeError):
-    raise ImportError('Error initializing ctypes')
+    raise ImportError("Error initializing ctypes")
 
 
 class CFConst(object):
@@ -502,6 +399,7 @@ class CFConst(object):
     A class object that acts as essentially a namespace for CoreFoundation
     constants.
     """
+
     kCFStringEncodingUTF8 = CFStringEncoding(0x08000100)
 
 
@@ -509,6 +407,7 @@ class SecurityConst(object):
     """
     A class object that acts as essentially a namespace for Security constants.
     """
+
     kSSLSessionOptionBreakOnServerAuth = 0
 
     kSSLProtocol2 = 1
@@ -516,6 +415,8 @@ class SecurityConst(object):
     kTLSProtocol1 = 4
     kTLSProtocol11 = 7
     kTLSProtocol12 = 8
+    kTLSProtocol13 = 10
+    kTLSProtocolMaxSupported = 999
 
     kSSLClientSide = 1
     kSSLStreamType = 0
@@ -558,30 +459,27 @@ class SecurityConst(object):
     errSecInvalidTrustSettings = -25262
 
     # Cipher suites. We only pick the ones our default cipher string allows.
+    # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values
     TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C
     TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030
     TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B
     TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F
-    TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3
+    TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9
+    TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8
     TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F
-    TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2
     TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E
     TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024
     TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028
     TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A
     TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014
     TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B
-    TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A
     TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039
-    TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038
     TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023
     TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027
     TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009
     TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013
     TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067
-    TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040
     TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033
-    TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032
     TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D
     TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C
     TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D
@@ -590,4 +488,5 @@ class SecurityConst(object):
     TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F
     TLS_AES_128_GCM_SHA256 = 0x1301
     TLS_AES_256_GCM_SHA384 = 0x1302
-    TLS_CHACHA20_POLY1305_SHA256 = 0x1303
+    TLS_AES_128_CCM_8_SHA256 = 0x1305
+    TLS_AES_128_CCM_SHA256 = 0x1304
diff --git a/lib/urllib3/contrib/_securetransport/low_level.py b/lib/urllib3/contrib/_securetransport/low_level.py
index 5e3494bc..e60168ca 100644
--- a/lib/urllib3/contrib/_securetransport/low_level.py
+++ b/lib/urllib3/contrib/_securetransport/low_level.py
@@ -66,22 +66,18 @@ def _cf_string_to_unicode(value):
     value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p))
 
     string = CoreFoundation.CFStringGetCStringPtr(
-        value_as_void_p,
-        CFConst.kCFStringEncodingUTF8
+        value_as_void_p, CFConst.kCFStringEncodingUTF8
     )
     if string is None:
         buffer = ctypes.create_string_buffer(1024)
         result = CoreFoundation.CFStringGetCString(
-            value_as_void_p,
-            buffer,
-            1024,
-            CFConst.kCFStringEncodingUTF8
+            value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8
         )
         if not result:
-            raise OSError('Error copying C string from CFStringRef')
+            raise OSError("Error copying C string from CFStringRef")
         string = buffer.value
     if string is not None:
-        string = string.decode('utf-8')
+        string = string.decode("utf-8")
     return string
 
 
@@ -97,8 +93,8 @@ def _assert_no_error(error, exception_class=None):
     output = _cf_string_to_unicode(cf_error_string)
     CoreFoundation.CFRelease(cf_error_string)
 
-    if output is None or output == u'':
-        output = u'OSStatus %s' % error
+    if output is None or output == u"":
+        output = u"OSStatus %s" % error
 
     if exception_class is None:
         exception_class = ssl.SSLError
@@ -111,9 +107,11 @@ def _cert_array_from_pem(pem_bundle):
     Given a bundle of certs in PEM format, turns them into a CFArray of certs
     that can be used to validate a cert chain.
     """
+    # Normalize the PEM bundle's line endings.
+    pem_bundle = pem_bundle.replace(b"\r\n", b"\n")
+
     der_certs = [
-        base64.b64decode(match.group(1))
-        for match in _PEM_CERTS_RE.finditer(pem_bundle)
+        base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle)
     ]
     if not der_certs:
         raise ssl.SSLError("No root certificates specified")
@@ -121,7 +119,7 @@ def _cert_array_from_pem(pem_bundle):
     cert_array = CoreFoundation.CFArrayCreateMutable(
         CoreFoundation.kCFAllocatorDefault,
         0,
-        ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks)
+        ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks),
     )
     if not cert_array:
         raise ssl.SSLError("Unable to allocate memory!")
@@ -183,21 +181,16 @@ def _temporary_keychain():
     # some random bytes to password-protect the keychain we're creating, so we
     # ask for 40 random bytes.
     random_bytes = os.urandom(40)
-    filename = base64.b64encode(random_bytes[:8]).decode('utf-8')
-    password = base64.b64encode(random_bytes[8:])  # Must be valid UTF-8
+    filename = base64.b16encode(random_bytes[:8]).decode("utf-8")
+    password = base64.b16encode(random_bytes[8:])  # Must be valid UTF-8
     tempdirectory = tempfile.mkdtemp()
 
-    keychain_path = os.path.join(tempdirectory, filename).encode('utf-8')
+    keychain_path = os.path.join(tempdirectory, filename).encode("utf-8")
 
     # We now want to create the keychain itself.
     keychain = Security.SecKeychainRef()
     status = Security.SecKeychainCreate(
-        keychain_path,
-        len(password),
-        password,
-        False,
-        None,
-        ctypes.byref(keychain)
+        keychain_path, len(password), password, False, None, ctypes.byref(keychain)
     )
     _assert_no_error(status)
 
@@ -216,14 +209,12 @@ def _load_items_from_file(keychain, path):
     identities = []
     result_array = None
 
-    with open(path, 'rb') as f:
+    with open(path, "rb") as f:
         raw_filedata = f.read()
 
     try:
         filedata = CoreFoundation.CFDataCreate(
-            CoreFoundation.kCFAllocatorDefault,
-            raw_filedata,
-            len(raw_filedata)
+            CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata)
         )
         result_array = CoreFoundation.CFArrayRef()
         result = Security.SecItemImport(
@@ -234,7 +225,7 @@ def _load_items_from_file(keychain, path):
             0,  # import flags
             None,  # key params, can include passphrase in the future
             keychain,  # The keychain to insert into
-            ctypes.byref(result_array)  # Results
+            ctypes.byref(result_array),  # Results
         )
         _assert_no_error(result)
 
@@ -244,9 +235,7 @@ def _load_items_from_file(keychain, path):
         # keychain already has them!
         result_count = CoreFoundation.CFArrayGetCount(result_array)
         for index in range(result_count):
-            item = CoreFoundation.CFArrayGetValueAtIndex(
-                result_array, index
-            )
+            item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index)
             item = ctypes.cast(item, CoreFoundation.CFTypeRef)
 
             if _is_cert(item):
@@ -304,9 +293,7 @@ def _load_client_cert_chain(keychain, *paths):
 
     try:
         for file_path in paths:
-            new_identities, new_certs = _load_items_from_file(
-                keychain, file_path
-            )
+            new_identities, new_certs = _load_items_from_file(keychain, file_path)
             identities.extend(new_identities)
             certificates.extend(new_certs)
 
@@ -315,9 +302,7 @@ def _load_client_cert_chain(keychain, *paths):
         if not identities:
             new_identity = Security.SecIdentityRef()
             status = Security.SecIdentityCreateWithCertificate(
-                keychain,
-                certificates[0],
-                ctypes.byref(new_identity)
+                keychain, certificates[0], ctypes.byref(new_identity)
             )
             _assert_no_error(status)
             identities.append(new_identity)
diff --git a/lib/urllib3/contrib/appengine.py b/lib/urllib3/contrib/appengine.py
index 814b0222..01c91409 100644
--- a/lib/urllib3/contrib/appengine.py
+++ b/lib/urllib3/contrib/appengine.py
@@ -39,8 +39,8 @@ urllib3 on Google App Engine:
 """
 
 from __future__ import absolute_import
+import io
 import logging
-import os
 import warnings
 from ..packages.six.moves.urllib.parse import urljoin
 
@@ -50,14 +50,14 @@ from ..exceptions import (
     MaxRetryError,
     ProtocolError,
     TimeoutError,
-    SSLError
+    SSLError,
 )
 
-from ..packages.six import BytesIO
 from ..request import RequestMethods
 from ..response import HTTPResponse
 from ..util.timeout import Timeout
 from ..util.retry import Retry
+from . import _appengine_environ
 
 try:
     from google.appengine.api import urlfetch
@@ -96,23 +96,31 @@ class AppEngineManager(RequestMethods):
     Beyond those cases, it will raise normal urllib3 errors.
     """
 
-    def __init__(self, headers=None, retries=None, validate_certificate=True,
-                 urlfetch_retries=True):
+    def __init__(
+        self,
+        headers=None,
+        retries=None,
+        validate_certificate=True,
+        urlfetch_retries=True,
+    ):
         if not urlfetch:
             raise AppEnginePlatformError(
-                "URLFetch is not available in this environment.")
+                "URLFetch is not available in this environment."
+            )
 
         if is_prod_appengine_mvms():
             raise AppEnginePlatformError(
                 "Use normal urllib3.PoolManager instead of AppEngineManager"
                 "on Managed VMs, as using URLFetch is not necessary in "
-                "this environment.")
+                "this environment."
+            )
 
         warnings.warn(
             "urllib3 is using URLFetch on Google App Engine sandbox instead "
             "of sockets. To use sockets directly instead of URLFetch see "
             "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.",
-            AppEnginePlatformWarning)
+            AppEnginePlatformWarning,
+        )
 
         RequestMethods.__init__(self, headers)
         self.validate_certificate = validate_certificate
@@ -127,17 +135,22 @@ class AppEngineManager(RequestMethods):
         # Return False to re-raise any potential exceptions
         return False
 
-    def urlopen(self, method, url, body=None, headers=None,
-                retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT,
-                **response_kw):
+    def urlopen(
+        self,
+        method,
+        url,
+        body=None,
+        headers=None,
+        retries=None,
+        redirect=True,
+        timeout=Timeout.DEFAULT_TIMEOUT,
+        **response_kw
+    ):
 
         retries = self._get_retries(retries, redirect)
 
         try:
-            follow_redirects = (
-                    redirect and
-                    retries.redirect != 0 and
-                    retries.total)
+            follow_redirects = redirect and retries.redirect != 0 and retries.total
             response = urlfetch.fetch(
                 url,
                 payload=body,
@@ -152,44 +165,52 @@ class AppEngineManager(RequestMethods):
             raise TimeoutError(self, e)
 
         except urlfetch.InvalidURLError as e:
-            if 'too large' in str(e):
+            if "too large" in str(e):
                 raise AppEnginePlatformError(
                     "URLFetch request too large, URLFetch only "
-                    "supports requests up to 10mb in size.", e)
+                    "supports requests up to 10mb in size.",
+                    e,
+                )
             raise ProtocolError(e)
 
         except urlfetch.DownloadError as e:
-            if 'Too many redirects' in str(e):
+            if "Too many redirects" in str(e):
                 raise MaxRetryError(self, url, reason=e)
             raise ProtocolError(e)
 
         except urlfetch.ResponseTooLargeError as e:
             raise AppEnginePlatformError(
                 "URLFetch response too large, URLFetch only supports"
-                "responses up to 32mb in size.", e)
+                "responses up to 32mb in size.",
+                e,
+            )
 
         except urlfetch.SSLCertificateError as e:
             raise SSLError(e)
 
         except urlfetch.InvalidMethodError as e:
             raise AppEnginePlatformError(
-                "URLFetch does not support method: %s" % method, e)
+                "URLFetch does not support method: %s" % method, e
+            )
 
         http_response = self._urlfetch_response_to_http_response(
-            response, retries=retries, **response_kw)
+            response, retries=retries, **response_kw
+        )
 
         # Handle redirect?
         redirect_location = redirect and http_response.get_redirect_location()
         if redirect_location:
             # Check for redirect response
-            if (self.urlfetch_retries and retries.raise_on_redirect):
+            if self.urlfetch_retries and retries.raise_on_redirect:
                 raise MaxRetryError(self, url, "too many redirects")
             else:
                 if http_response.status == 303:
-                    method = 'GET'
+                    method = "GET"
 
                 try:
-                    retries = retries.increment(method, url, response=http_response, _pool=self)
+                    retries = retries.increment(
+                        method, url, response=http_response, _pool=self
+                    )
                 except MaxRetryError:
                     if retries.raise_on_redirect:
                         raise MaxRetryError(self, url, "too many redirects")
@@ -199,22 +220,32 @@ class AppEngineManager(RequestMethods):
                 log.debug("Redirecting %s -> %s", url, redirect_location)
                 redirect_url = urljoin(url, redirect_location)
                 return self.urlopen(
-                    method, redirect_url, body, headers,
-                    retries=retries, redirect=redirect,
-                    timeout=timeout, **response_kw)
+                    method,
+                    redirect_url,
+                    body,
+                    headers,
+                    retries=retries,
+                    redirect=redirect,
+                    timeout=timeout,
+                    **response_kw
+                )
 
         # Check if we should retry the HTTP response.
-        has_retry_after = bool(http_response.getheader('Retry-After'))
+        has_retry_after = bool(http_response.getheader("Retry-After"))
         if retries.is_retry(method, http_response.status, has_retry_after):
-            retries = retries.increment(
-                method, url, response=http_response, _pool=self)
+            retries = retries.increment(method, url, response=http_response, _pool=self)
             log.debug("Retry: %s", url)
             retries.sleep(http_response)
             return self.urlopen(
-                method, url,
-                body=body, headers=headers,
-                retries=retries, redirect=redirect,
-                timeout=timeout, **response_kw)
+                method,
+                url,
+                body=body,
+                headers=headers,
+                retries=retries,
+                redirect=redirect,
+                timeout=timeout,
+                **response_kw
+            )
 
         return http_response
 
@@ -223,28 +254,37 @@ class AppEngineManager(RequestMethods):
         if is_prod_appengine():
             # Production GAE handles deflate encoding automatically, but does
             # not remove the encoding header.
-            content_encoding = urlfetch_resp.headers.get('content-encoding')
+            content_encoding = urlfetch_resp.headers.get("content-encoding")
 
-            if content_encoding == 'deflate':
-                del urlfetch_resp.headers['content-encoding']
+            if content_encoding == "deflate":
+                del urlfetch_resp.headers["content-encoding"]
 
-        transfer_encoding = urlfetch_resp.headers.get('transfer-encoding')
+        transfer_encoding = urlfetch_resp.headers.get("transfer-encoding")
         # We have a full response's content,
         # so let's make sure we don't report ourselves as chunked data.
-        if transfer_encoding == 'chunked':
+        if transfer_encoding == "chunked":
             encodings = transfer_encoding.split(",")
-            encodings.remove('chunked')
-            urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings)
+            encodings.remove("chunked")
+            urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings)
 
-        return HTTPResponse(
+        original_response = HTTPResponse(
             # In order for decoding to work, we must present the content as
             # a file-like object.
-            body=BytesIO(urlfetch_resp.content),
+            body=io.BytesIO(urlfetch_resp.content),
+            msg=urlfetch_resp.header_msg,
             headers=urlfetch_resp.headers,
             status=urlfetch_resp.status_code,
             **response_kw
         )
 
+        return HTTPResponse(
+            body=io.BytesIO(urlfetch_resp.content),
+            headers=urlfetch_resp.headers,
+            status=urlfetch_resp.status_code,
+            original_response=original_response,
+            **response_kw
+        )
+
     def _get_absolute_timeout(self, timeout):
         if timeout is Timeout.DEFAULT_TIMEOUT:
             return None  # Defer to URLFetch's default.
@@ -253,44 +293,29 @@ class AppEngineManager(RequestMethods):
                 warnings.warn(
                     "URLFetch does not support granular timeout settings, "
                     "reverting to total or default URLFetch timeout.",
-                    AppEnginePlatformWarning)
+                    AppEnginePlatformWarning,
+                )
             return timeout.total
         return timeout
 
     def _get_retries(self, retries, redirect):
         if not isinstance(retries, Retry):
-            retries = Retry.from_int(
-                retries, redirect=redirect, default=self.retries)
+            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
 
         if retries.connect or retries.read or retries.redirect:
             warnings.warn(
                 "URLFetch only supports total retries and does not "
                 "recognize connect, read, or redirect retry parameters.",
-                AppEnginePlatformWarning)
+                AppEnginePlatformWarning,
+            )
 
         return retries
 
 
-def is_appengine():
-    return (is_local_appengine() or
-            is_prod_appengine() or
-            is_prod_appengine_mvms())
+# Alias methods from _appengine_environ to maintain public API interface.
 
-
-def is_appengine_sandbox():
-    return is_appengine() and not is_prod_appengine_mvms()
-
-
-def is_local_appengine():
-    return ('APPENGINE_RUNTIME' in os.environ and
-            'Development/' in os.environ['SERVER_SOFTWARE'])
-
-
-def is_prod_appengine():
-    return ('APPENGINE_RUNTIME' in os.environ and
-            'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and
-            not is_prod_appengine_mvms())
-
-
-def is_prod_appengine_mvms():
-    return os.environ.get('GAE_VM', False) == 'true'
+is_appengine = _appengine_environ.is_appengine
+is_appengine_sandbox = _appengine_environ.is_appengine_sandbox
+is_local_appengine = _appengine_environ.is_local_appengine
+is_prod_appengine = _appengine_environ.is_prod_appengine
+is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms
diff --git a/lib/urllib3/contrib/ntlmpool.py b/lib/urllib3/contrib/ntlmpool.py
index 642e99ed..9c96be29 100644
--- a/lib/urllib3/contrib/ntlmpool.py
+++ b/lib/urllib3/contrib/ntlmpool.py
@@ -20,7 +20,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
     Implements an NTLM authentication version of an urllib3 connection pool
     """
 
-    scheme = 'https'
+    scheme = "https"
 
     def __init__(self, user, pw, authurl, *args, **kwargs):
         """
@@ -31,7 +31,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
         super(NTLMConnectionPool, self).__init__(*args, **kwargs)
         self.authurl = authurl
         self.rawuser = user
-        user_parts = user.split('\\', 1)
+        user_parts = user.split("\\", 1)
         self.domain = user_parts[0].upper()
         self.user = user_parts[1]
         self.pw = pw
@@ -40,73 +40,84 @@ class NTLMConnectionPool(HTTPSConnectionPool):
         # Performs the NTLM handshake that secures the connection. The socket
         # must be kept open while requests are performed.
         self.num_connections += 1
-        log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
-                  self.num_connections, self.host, self.authurl)
+        log.debug(
+            "Starting NTLM HTTPS connection no. %d: https://%s%s",
+            self.num_connections,
+            self.host,
+            self.authurl,
+        )
 
-        headers = {}
-        headers['Connection'] = 'Keep-Alive'
-        req_header = 'Authorization'
-        resp_header = 'www-authenticate'
+        headers = {"Connection": "Keep-Alive"}
+        req_header = "Authorization"
+        resp_header = "www-authenticate"
 
         conn = HTTPSConnection(host=self.host, port=self.port)
 
         # Send negotiation message
-        headers[req_header] = (
-            'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
-        log.debug('Request headers: %s', headers)
-        conn.request('GET', self.authurl, None, headers)
+        headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE(
+            self.rawuser
+        )
+        log.debug("Request headers: %s", headers)
+        conn.request("GET", self.authurl, None, headers)
         res = conn.getresponse()
         reshdr = dict(res.getheaders())
-        log.debug('Response status: %s %s', res.status, res.reason)
-        log.debug('Response headers: %s', reshdr)
-        log.debug('Response data: %s [...]', res.read(100))
+        log.debug("Response status: %s %s", res.status, res.reason)
+        log.debug("Response headers: %s", reshdr)
+        log.debug("Response data: %s [...]", res.read(100))
 
         # Remove the reference to the socket, so that it can not be closed by
         # the response object (we want to keep the socket open)
         res.fp = None
 
         # Server should respond with a challenge message
-        auth_header_values = reshdr[resp_header].split(', ')
+        auth_header_values = reshdr[resp_header].split(", ")
         auth_header_value = None
         for s in auth_header_values:
-            if s[:5] == 'NTLM ':
+            if s[:5] == "NTLM ":
                 auth_header_value = s[5:]
         if auth_header_value is None:
-            raise Exception('Unexpected %s response header: %s' %
-                            (resp_header, reshdr[resp_header]))
+            raise Exception(
+                "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header])
+            )
 
         # Send authentication message
-        ServerChallenge, NegotiateFlags = \
-            ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
-        auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge,
-                                                         self.user,
-                                                         self.domain,
-                                                         self.pw,
-                                                         NegotiateFlags)
-        headers[req_header] = 'NTLM %s' % auth_msg
-        log.debug('Request headers: %s', headers)
-        conn.request('GET', self.authurl, None, headers)
+        ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
+            auth_header_value
+        )
+        auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
+            ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags
+        )
+        headers[req_header] = "NTLM %s" % auth_msg
+        log.debug("Request headers: %s", headers)
+        conn.request("GET", self.authurl, None, headers)
         res = conn.getresponse()
-        log.debug('Response status: %s %s', res.status, res.reason)
-        log.debug('Response headers: %s', dict(res.getheaders()))
-        log.debug('Response data: %s [...]', res.read()[:100])
+        log.debug("Response status: %s %s", res.status, res.reason)
+        log.debug("Response headers: %s", dict(res.getheaders()))
+        log.debug("Response data: %s [...]", res.read()[:100])
         if res.status != 200:
             if res.status == 401:
-                raise Exception('Server rejected request: wrong '
-                                'username or password')
-            raise Exception('Wrong server response: %s %s' %
-                            (res.status, res.reason))
+                raise Exception(
+                    "Server rejected request: wrong " "username or password"
+                )
+            raise Exception("Wrong server response: %s %s" % (res.status, res.reason))
 
         res.fp = None
-        log.debug('Connection established')
+        log.debug("Connection established")
         return conn
 
-    def urlopen(self, method, url, body=None, headers=None, retries=3,
-                redirect=True, assert_same_host=True):
+    def urlopen(
+        self,
+        method,
+        url,
+        body=None,
+        headers=None,
+        retries=3,
+        redirect=True,
+        assert_same_host=True,
+    ):
         if headers is None:
             headers = {}
-        headers['Connection'] = 'Keep-Alive'
-        return super(NTLMConnectionPool, self).urlopen(method, url, body,
-                                                       headers, retries,
-                                                       redirect,
-                                                       assert_same_host)
+        headers["Connection"] = "Keep-Alive"
+        return super(NTLMConnectionPool, self).urlopen(
+            method, url, body, headers, retries, redirect, assert_same_host
+        )
diff --git a/lib/urllib3/contrib/pyopenssl.py b/lib/urllib3/contrib/pyopenssl.py
new file mode 100644
index 00000000..3051ef3a
--- /dev/null
+++ b/lib/urllib3/contrib/pyopenssl.py
@@ -0,0 +1,498 @@
+"""
+SSL with SNI_-support for Python 2. Follow these instructions if you would
+like to verify SSL certificates in Python 2. Note, the default libraries do
+*not* do certificate checking; you need to do additional work to validate
+certificates yourself.
+
+This needs the following packages installed:
+
+* pyOpenSSL (tested with 16.0.0)
+* cryptography (minimum 1.3.4, from pyopenssl)
+* idna (minimum 2.0, from cryptography)
+
+However, pyopenssl depends on cryptography, which depends on idna, so while we
+use all three directly here we end up having relatively few packages required.
+
+You can install them with the following command:
+
+    pip install pyopenssl cryptography idna
+
+To activate certificate checking, call
+:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
+before you begin making HTTP requests. This can be done in a ``sitecustomize``
+module, or at any other time before your application begins using ``urllib3``,
+like this::
+
+    try:
+        import urllib3.contrib.pyopenssl
+        urllib3.contrib.pyopenssl.inject_into_urllib3()
+    except ImportError:
+        pass
+
+Now you can use :mod:`urllib3` as you normally would, and it will support SNI
+when the required modules are installed.
+
+Activating this module also has the positive side effect of disabling SSL/TLS
+compression in Python 2 (see `CRIME attack`_).
+
+If you want to configure the default list of supported cipher suites, you can
+set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
+
+.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
+.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
+"""
+from __future__ import absolute_import
+
+import OpenSSL.SSL
+from cryptography import x509
+from cryptography.hazmat.backends.openssl import backend as openssl_backend
+from cryptography.hazmat.backends.openssl.x509 import _Certificate
+
+try:
+    from cryptography.x509 import UnsupportedExtension
+except ImportError:
+    # UnsupportedExtension is gone in cryptography >= 2.1.0
+    class UnsupportedExtension(Exception):
+        pass
+
+
+from socket import timeout, error as SocketError
+from io import BytesIO
+
+try:  # Platform-specific: Python 2
+    from socket import _fileobject
+except ImportError:  # Platform-specific: Python 3
+    _fileobject = None
+    from ..packages.backports.makefile import backport_makefile
+
+import logging
+import ssl
+from ..packages import six
+import sys
+
+from .. import util
+
+
+__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
+
+# SNI always works.
+HAS_SNI = True
+
+# Map from urllib3 to PyOpenSSL compatible parameter-values.
+_openssl_versions = {
+    util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD,
+    ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
+}
+
+if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"):
+    _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD
+
+if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
+    _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
+
+if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
+    _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
+
+
+_stdlib_to_openssl_verify = {
+    ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
+    ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
+    ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
+    + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
+}
+_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items())
+
+# OpenSSL will only write 16K at a time
+SSL_WRITE_BLOCKSIZE = 16384
+
+orig_util_HAS_SNI = util.HAS_SNI
+orig_util_SSLContext = util.ssl_.SSLContext
+
+
+log = logging.getLogger(__name__)
+
+
+def inject_into_urllib3():
+    "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
+
+    _validate_dependencies_met()
+
+    util.SSLContext = PyOpenSSLContext
+    util.ssl_.SSLContext = PyOpenSSLContext
+    util.HAS_SNI = HAS_SNI
+    util.ssl_.HAS_SNI = HAS_SNI
+    util.IS_PYOPENSSL = True
+    util.ssl_.IS_PYOPENSSL = True
+
+
+def extract_from_urllib3():
+    "Undo monkey-patching by :func:`inject_into_urllib3`."
+
+    util.SSLContext = orig_util_SSLContext
+    util.ssl_.SSLContext = orig_util_SSLContext
+    util.HAS_SNI = orig_util_HAS_SNI
+    util.ssl_.HAS_SNI = orig_util_HAS_SNI
+    util.IS_PYOPENSSL = False
+    util.ssl_.IS_PYOPENSSL = False
+
+
+def _validate_dependencies_met():
+    """
+    Verifies that PyOpenSSL's package-level dependencies have been met.
+    Throws `ImportError` if they are not met.
+    """
+    # Method added in `cryptography==1.1`; not available in older versions
+    from cryptography.x509.extensions import Extensions
+
+    if getattr(Extensions, "get_extension_for_class", None) is None:
+        raise ImportError(
+            "'cryptography' module missing required functionality.  "
+            "Try upgrading to v1.3.4 or newer."
+        )
+
+    # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
+    # attribute is only present on those versions.
+    from OpenSSL.crypto import X509
+
+    x509 = X509()
+    if getattr(x509, "_x509", None) is None:
+        raise ImportError(
+            "'pyOpenSSL' module missing required functionality. "
+            "Try upgrading to v0.14 or newer."
+        )
+
+
+def _dnsname_to_stdlib(name):
+    """
+    Converts a dNSName SubjectAlternativeName field to the form used by the
+    standard library on the given Python version.
+
+    Cryptography produces a dNSName as a unicode string that was idna-decoded
+    from ASCII bytes. We need to idna-encode that string to get it back, and
+    then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
+    uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
+
+    If the name cannot be idna-encoded then we return None signalling that
+    the name given should be skipped.
+    """
+
+    def idna_encode(name):
+        """
+        Borrowed wholesale from the Python Cryptography Project. It turns out
+        that we can't just safely call `idna.encode`: it can explode for
+        wildcard names. This avoids that problem.
+        """
+        import idna
+
+        try:
+            for prefix in [u"*.", u"."]:
+                if name.startswith(prefix):
+                    name = name[len(prefix) :]
+                    return prefix.encode("ascii") + idna.encode(name)
+            return idna.encode(name)
+        except idna.core.IDNAError:
+            return None
+
+    # Don't send IPv6 addresses through the IDNA encoder.
+    if ":" in name:
+        return name
+
+    name = idna_encode(name)
+    if name is None:
+        return None
+    elif sys.version_info >= (3, 0):
+        name = name.decode("utf-8")
+    return name
+
+
+def get_subj_alt_name(peer_cert):
+    """
+    Given an PyOpenSSL certificate, provides all the subject alternative names.
+    """
+    # Pass the cert to cryptography, which has much better APIs for this.
+    if hasattr(peer_cert, "to_cryptography"):
+        cert = peer_cert.to_cryptography()
+    else:
+        # This is technically using private APIs, but should work across all
+        # relevant versions before PyOpenSSL got a proper API for this.
+        cert = _Certificate(openssl_backend, peer_cert._x509)
+
+    # We want to find the SAN extension. Ask Cryptography to locate it (it's
+    # faster than looping in Python)
+    try:
+        ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
+    except x509.ExtensionNotFound:
+        # No such extension, return the empty list.
+        return []
+    except (
+        x509.DuplicateExtension,
+        UnsupportedExtension,
+        x509.UnsupportedGeneralNameType,
+        UnicodeError,
+    ) as e:
+        # A problem has been found with the quality of the certificate. Assume
+        # no SAN field is present.
+        log.warning(
+            "A problem was encountered with the certificate that prevented "
+            "urllib3 from finding the SubjectAlternativeName field. This can "
+            "affect certificate validation. The error was %s",
+            e,
+        )
+        return []
+
+    # We want to return dNSName and iPAddress fields. We need to cast the IPs
+    # back to strings because the match_hostname function wants them as
+    # strings.
+    # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
+    # decoded. This is pretty frustrating, but that's what the standard library
+    # does with certificates, and so we need to attempt to do the same.
+    # We also want to skip over names which cannot be idna encoded.
+    names = [
+        ("DNS", name)
+        for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
+        if name is not None
+    ]
+    names.extend(
+        ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
+    )
+
+    return names
+
+
+class WrappedSocket(object):
+    """API-compatibility wrapper for Python OpenSSL's Connection-class.
+
+    Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
+    collector of pypy.
+    """
+
+    def __init__(self, connection, socket, suppress_ragged_eofs=True):
+        self.connection = connection
+        self.socket = socket
+        self.suppress_ragged_eofs = suppress_ragged_eofs
+        self._makefile_refs = 0
+        self._closed = False
+
+    def fileno(self):
+        return self.socket.fileno()
+
+    # Copy-pasted from Python 3.5 source code
+    def _decref_socketios(self):
+        if self._makefile_refs > 0:
+            self._makefile_refs -= 1
+        if self._closed:
+            self.close()
+
+    def recv(self, *args, **kwargs):
+        try:
+            data = self.connection.recv(*args, **kwargs)
+        except OpenSSL.SSL.SysCallError as e:
+            if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
+                return b""
+            else:
+                raise SocketError(str(e))
+        except OpenSSL.SSL.ZeroReturnError:
+            if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+                return b""
+            else:
+                raise
+        except OpenSSL.SSL.WantReadError:
+            if not util.wait_for_read(self.socket, self.socket.gettimeout()):
+                raise timeout("The read operation timed out")
+            else:
+                return self.recv(*args, **kwargs)
+
+        # TLS 1.3 post-handshake authentication
+        except OpenSSL.SSL.Error as e:
+            raise ssl.SSLError("read error: %r" % e)
+        else:
+            return data
+
+    def recv_into(self, *args, **kwargs):
+        try:
+            return self.connection.recv_into(*args, **kwargs)
+        except OpenSSL.SSL.SysCallError as e:
+            if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
+                return 0
+            else:
+                raise SocketError(str(e))
+        except OpenSSL.SSL.ZeroReturnError:
+            if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+                return 0
+            else:
+                raise
+        except OpenSSL.SSL.WantReadError:
+            if not util.wait_for_read(self.socket, self.socket.gettimeout()):
+                raise timeout("The read operation timed out")
+            else:
+                return self.recv_into(*args, **kwargs)
+
+        # TLS 1.3 post-handshake authentication
+        except OpenSSL.SSL.Error as e:
+            raise ssl.SSLError("read error: %r" % e)
+
+    def settimeout(self, timeout):
+        return self.socket.settimeout(timeout)
+
+    def _send_until_done(self, data):
+        while True:
+            try:
+                return self.connection.send(data)
+            except OpenSSL.SSL.WantWriteError:
+                if not util.wait_for_write(self.socket, self.socket.gettimeout()):
+                    raise timeout()
+                continue
+            except OpenSSL.SSL.SysCallError as e:
+                raise SocketError(str(e))
+
+    def sendall(self, data):
+        total_sent = 0
+        while total_sent < len(data):
+            sent = self._send_until_done(
+                data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
+            )
+            total_sent += sent
+
+    def shutdown(self):
+        # FIXME rethrow compatible exceptions should we ever use this
+        self.connection.shutdown()
+
+    def close(self):
+        if self._makefile_refs < 1:
+            try:
+                self._closed = True
+                return self.connection.close()
+            except OpenSSL.SSL.Error:
+                return
+        else:
+            self._makefile_refs -= 1
+
+    def getpeercert(self, binary_form=False):
+        x509 = self.connection.get_peer_certificate()
+
+        if not x509:
+            return x509
+
+        if binary_form:
+            return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509)
+
+        return {
+            "subject": ((("commonName", x509.get_subject().CN),),),
+            "subjectAltName": get_subj_alt_name(x509),
+        }
+
+    def version(self):
+        return self.connection.get_protocol_version_name()
+
+    def _reuse(self):
+        self._makefile_refs += 1
+
+    def _drop(self):
+        if self._makefile_refs < 1:
+            self.close()
+        else:
+            self._makefile_refs -= 1
+
+
+if _fileobject:  # Platform-specific: Python 2
+
+    def makefile(self, mode, bufsize=-1):
+        self._makefile_refs += 1
+        return _fileobject(self, mode, bufsize, close=True)
+
+
+else:  # Platform-specific: Python 3
+    makefile = backport_makefile
+
+WrappedSocket.makefile = makefile
+
+
+class PyOpenSSLContext(object):
+    """
+    I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
+    for translating the interface of the standard library ``SSLContext`` object
+    to calls into PyOpenSSL.
+    """
+
+    def __init__(self, protocol):
+        self.protocol = _openssl_versions[protocol]
+        self._ctx = OpenSSL.SSL.Context(self.protocol)
+        self._options = 0
+        self.check_hostname = False
+
+    @property
+    def options(self):
+        return self._options
+
+    @options.setter
+    def options(self, value):
+        self._options = value
+        self._ctx.set_options(value)
+
+    @property
+    def verify_mode(self):
+        return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
+
+    @verify_mode.setter
+    def verify_mode(self, value):
+        self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
+
+    def set_default_verify_paths(self):
+        self._ctx.set_default_verify_paths()
+
+    def set_ciphers(self, ciphers):
+        if isinstance(ciphers, six.text_type):
+            ciphers = ciphers.encode("utf-8")
+        self._ctx.set_cipher_list(ciphers)
+
+    def load_verify_locations(self, cafile=None, capath=None, cadata=None):
+        if cafile is not None:
+            cafile = cafile.encode("utf-8")
+        if capath is not None:
+            capath = capath.encode("utf-8")
+        self._ctx.load_verify_locations(cafile, capath)
+        if cadata is not None:
+            self._ctx.load_verify_locations(BytesIO(cadata))
+
+    def load_cert_chain(self, certfile, keyfile=None, password=None):
+        self._ctx.use_certificate_chain_file(certfile)
+        if password is not None:
+            if not isinstance(password, six.binary_type):
+                password = password.encode("utf-8")
+            self._ctx.set_passwd_cb(lambda *_: password)
+        self._ctx.use_privatekey_file(keyfile or certfile)
+
+    def wrap_socket(
+        self,
+        sock,
+        server_side=False,
+        do_handshake_on_connect=True,
+        suppress_ragged_eofs=True,
+        server_hostname=None,
+    ):
+        cnx = OpenSSL.SSL.Connection(self._ctx, sock)
+
+        if isinstance(server_hostname, six.text_type):  # Platform-specific: Python 3
+            server_hostname = server_hostname.encode("utf-8")
+
+        if server_hostname is not None:
+            cnx.set_tlsext_host_name(server_hostname)
+
+        cnx.set_connect_state()
+
+        while True:
+            try:
+                cnx.do_handshake()
+            except OpenSSL.SSL.WantReadError:
+                if not util.wait_for_read(sock, sock.gettimeout()):
+                    raise timeout("select timed out")
+                continue
+            except OpenSSL.SSL.Error as e:
+                raise ssl.SSLError("bad handshake: %r" % e)
+            break
+
+        return WrappedSocket(cnx, sock)
+
+
+def _verify_callback(cnx, x509, err_no, err_depth, return_code):
+    return err_no == 0
diff --git a/lib/urllib3/contrib/securetransport.py b/lib/urllib3/contrib/securetransport.py
index 2cac70f7..24e6b5c4 100644
--- a/lib/urllib3/contrib/securetransport.py
+++ b/lib/urllib3/contrib/securetransport.py
@@ -23,6 +23,31 @@ To use this module, simply import and inject it::
     urllib3.contrib.securetransport.inject_into_urllib3()
 
 Happy TLSing!
+
+This code is a bastardised version of the code found in Will Bond's oscrypto
+library. An enormous debt is owed to him for blazing this trail for us. For
+that reason, this code should be considered to be covered both by urllib3's
+license and by oscrypto's:
+
+    Copyright (c) 2015-2016 Will Bond 
+
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
 """
 from __future__ import absolute_import
 
@@ -37,12 +62,12 @@ import threading
 import weakref
 
 from .. import util
-from ._securetransport.bindings import (
-    Security, SecurityConst, CoreFoundation
-)
+from ._securetransport.bindings import Security, SecurityConst, CoreFoundation
 from ._securetransport.low_level import (
-    _assert_no_error, _cert_array_from_pem, _temporary_keychain,
-    _load_client_cert_chain
+    _assert_no_error,
+    _cert_array_from_pem,
+    _temporary_keychain,
+    _load_client_cert_chain,
 )
 
 try:  # Platform-specific: Python 2
@@ -51,12 +76,7 @@ except ImportError:  # Platform-specific: Python 3
     _fileobject = None
     from ..packages.backports.makefile import backport_makefile
 
-try:
-    memoryview(b'')
-except NameError:
-    raise ImportError("SecureTransport only works on Pythons with memoryview")
-
-__all__ = ['inject_into_urllib3', 'extract_from_urllib3']
+__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
 
 # SNI always works
 HAS_SNI = True
@@ -88,38 +108,35 @@ _connection_ref_lock = threading.Lock()
 SSL_WRITE_BLOCKSIZE = 16384
 
 # This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to
-# individual cipher suites. We need to do this becuase this is how
+# individual cipher suites. We need to do this because this is how
 # SecureTransport wants them.
 CIPHER_SUITES = [
-    SecurityConst.TLS_AES_256_GCM_SHA384,
-    SecurityConst.TLS_CHACHA20_POLY1305_SHA256,
-    SecurityConst.TLS_AES_128_GCM_SHA256,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
     SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384,
+    SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+    SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
     SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256,
     SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
-    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
+    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
     SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA,
+    SecurityConst.TLS_AES_256_GCM_SHA384,
+    SecurityConst.TLS_AES_128_GCM_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384,
     SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256,
+    SecurityConst.TLS_AES_128_CCM_8_SHA256,
+    SecurityConst.TLS_AES_128_CCM_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA,
@@ -127,39 +144,47 @@ CIPHER_SUITES = [
 ]
 
 # Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of
-# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version.
+# TLSv1 and a high of TLSv1.3. For everything else, we pin to that version.
+# TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+
 _protocol_to_min_max = {
-    ssl.PROTOCOL_SSLv23: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12),
+    util.PROTOCOL_TLS: (
+        SecurityConst.kTLSProtocol1,
+        SecurityConst.kTLSProtocolMaxSupported,
+    )
 }
 
 if hasattr(ssl, "PROTOCOL_SSLv2"):
     _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = (
-        SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2
+        SecurityConst.kSSLProtocol2,
+        SecurityConst.kSSLProtocol2,
     )
 if hasattr(ssl, "PROTOCOL_SSLv3"):
     _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = (
-        SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3
+        SecurityConst.kSSLProtocol3,
+        SecurityConst.kSSLProtocol3,
     )
 if hasattr(ssl, "PROTOCOL_TLSv1"):
     _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = (
-        SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1
+        SecurityConst.kTLSProtocol1,
+        SecurityConst.kTLSProtocol1,
     )
 if hasattr(ssl, "PROTOCOL_TLSv1_1"):
     _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = (
-        SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11
+        SecurityConst.kTLSProtocol11,
+        SecurityConst.kTLSProtocol11,
     )
 if hasattr(ssl, "PROTOCOL_TLSv1_2"):
     _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = (
-        SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12
+        SecurityConst.kTLSProtocol12,
+        SecurityConst.kTLSProtocol12,
     )
-if hasattr(ssl, "PROTOCOL_TLS"):
-    _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ssl.PROTOCOL_SSLv23]
 
 
 def inject_into_urllib3():
     """
     Monkey-patch urllib3 with SecureTransport-backed SSL-support.
     """
+    util.SSLContext = SecureTransportContext
     util.ssl_.SSLContext = SecureTransportContext
     util.HAS_SNI = HAS_SNI
     util.ssl_.HAS_SNI = HAS_SNI
@@ -171,6 +196,7 @@ def extract_from_urllib3():
     """
     Undo monkey-patching by :func:`inject_into_urllib3`.
     """
+    util.SSLContext = orig_util_SSLContext
     util.ssl_.SSLContext = orig_util_SSLContext
     util.HAS_SNI = orig_util_HAS_SNI
     util.ssl_.HAS_SNI = orig_util_HAS_SNI
@@ -195,21 +221,18 @@ def _read_callback(connection_id, data_buffer, data_length_pointer):
         timeout = wrapped_socket.gettimeout()
         error = None
         read_count = 0
-        buffer = (ctypes.c_char * requested_length).from_address(data_buffer)
-        buffer_view = memoryview(buffer)
 
         try:
             while read_count < requested_length:
                 if timeout is None or timeout >= 0:
-                    readables = util.wait_for_read([base_socket], timeout)
-                    if not readables:
-                        raise socket.error(errno.EAGAIN, 'timed out')
+                    if not util.wait_for_read(base_socket, timeout):
+                        raise socket.error(errno.EAGAIN, "timed out")
 
-                # We need to tell ctypes that we have a buffer that can be
-                # written to. Upsettingly, we do that like this:
-                chunk_size = base_socket.recv_into(
-                    buffer_view[read_count:requested_length]
+                remaining = requested_length - read_count
+                buffer = (ctypes.c_char * remaining).from_address(
+                    data_buffer + read_count
                 )
+                chunk_size = base_socket.recv_into(buffer, remaining)
                 read_count += chunk_size
                 if not chunk_size:
                     if not read_count:
@@ -219,7 +242,8 @@ def _read_callback(connection_id, data_buffer, data_length_pointer):
             error = e.errno
 
             if error is not None and error != errno.EAGAIN:
-                if error == errno.ECONNRESET:
+                data_length_pointer[0] = read_count
+                if error == errno.ECONNRESET or error == errno.EPIPE:
                     return SecurityConst.errSSLClosedAbort
                 raise
 
@@ -257,9 +281,8 @@ def _write_callback(connection_id, data_buffer, data_length_pointer):
         try:
             while sent < bytes_to_write:
                 if timeout is None or timeout >= 0:
-                    writables = util.wait_for_write([base_socket], timeout)
-                    if not writables:
-                        raise socket.error(errno.EAGAIN, 'timed out')
+                    if not util.wait_for_write(base_socket, timeout):
+                        raise socket.error(errno.EAGAIN, "timed out")
                 chunk_sent = base_socket.send(data)
                 sent += chunk_sent
 
@@ -270,11 +293,13 @@ def _write_callback(connection_id, data_buffer, data_length_pointer):
             error = e.errno
 
             if error is not None and error != errno.EAGAIN:
-                if error == errno.ECONNRESET:
+                data_length_pointer[0] = sent
+                if error == errno.ECONNRESET or error == errno.EPIPE:
                     return SecurityConst.errSSLClosedAbort
                 raise
 
         data_length_pointer[0] = sent
+
         if sent != bytes_to_write:
             return SecurityConst.errSSLWouldBlock
 
@@ -299,6 +324,7 @@ class WrappedSocket(object):
     Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage
     collector of PyPy.
     """
+
     def __init__(self, socket):
         self.socket = socket
         self.context = None
@@ -363,7 +389,7 @@ class WrappedSocket(object):
 
         # We want data in memory, so load it up.
         if os.path.isfile(trust_bundle):
-            with open(trust_bundle, 'rb') as f:
+            with open(trust_bundle, "rb") as f:
                 trust_bundle = f.read()
 
         cert_array = None
@@ -377,9 +403,7 @@ class WrappedSocket(object):
             # created for this connection, shove our CAs into it, tell ST to
             # ignore everything else it knows, and then ask if it can build a
             # chain. This is a buuuunch of code.
-            result = Security.SSLCopyPeerTrust(
-                self.context, ctypes.byref(trust)
-            )
+            result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust))
             _assert_no_error(result)
             if not trust:
                 raise ssl.SSLError("Failed to copy trust reference")
@@ -391,37 +415,36 @@ class WrappedSocket(object):
             _assert_no_error(result)
 
             trust_result = Security.SecTrustResultType()
-            result = Security.SecTrustEvaluate(
-                trust, ctypes.byref(trust_result)
-            )
+            result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result))
             _assert_no_error(result)
         finally:
             if trust:
                 CoreFoundation.CFRelease(trust)
 
-            if cert_array is None:
+            if cert_array is not None:
                 CoreFoundation.CFRelease(cert_array)
 
         # Ok, now we can look at what the result was.
         successes = (
             SecurityConst.kSecTrustResultUnspecified,
-            SecurityConst.kSecTrustResultProceed
+            SecurityConst.kSecTrustResultProceed,
         )
         if trust_result.value not in successes:
             raise ssl.SSLError(
-                "certificate verify failed, error code: %d" %
-                trust_result.value
+                "certificate verify failed, error code: %d" % trust_result.value
             )
 
-    def handshake(self,
-                  server_hostname,
-                  verify,
-                  trust_bundle,
-                  min_version,
-                  max_version,
-                  client_cert,
-                  client_key,
-                  client_key_passphrase):
+    def handshake(
+        self,
+        server_hostname,
+        verify,
+        trust_bundle,
+        min_version,
+        max_version,
+        client_cert,
+        client_key,
+        client_key_passphrase,
+    ):
         """
         Actually performs the TLS handshake. This is run automatically by
         wrapped socket, and shouldn't be needed in user code.
@@ -451,7 +474,7 @@ class WrappedSocket(object):
         # If we have a server hostname, we should set that too.
         if server_hostname:
             if not isinstance(server_hostname, bytes):
-                server_hostname = server_hostname.encode('utf-8')
+                server_hostname = server_hostname.encode("utf-8")
 
             result = Security.SSLSetPeerDomainName(
                 self.context, server_hostname, len(server_hostname)
@@ -464,7 +487,16 @@ class WrappedSocket(object):
         # Set the minimum and maximum TLS versions.
         result = Security.SSLSetProtocolVersionMin(self.context, min_version)
         _assert_no_error(result)
+
+        # TLS 1.3 isn't necessarily enabled by the OS
+        # so we have to detect when we error out and try
+        # setting TLS 1.3 if it's allowed. kTLSProtocolMaxSupported
+        # was added in macOS 10.13 along with kTLSProtocol13.
         result = Security.SSLSetProtocolVersionMax(self.context, max_version)
+        if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported:
+            result = Security.SSLSetProtocolVersionMax(
+                self.context, SecurityConst.kTLSProtocol12
+            )
         _assert_no_error(result)
 
         # If there's a trust DB, we need to use it. We do that by telling
@@ -473,9 +505,7 @@ class WrappedSocket(object):
         # authing in that case.
         if not verify or trust_bundle is not None:
             result = Security.SSLSetSessionOption(
-                self.context,
-                SecurityConst.kSSLSessionOptionBreakOnServerAuth,
-                True
+                self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True
             )
             _assert_no_error(result)
 
@@ -485,9 +515,7 @@ class WrappedSocket(object):
             self._client_cert_chain = _load_client_cert_chain(
                 self._keychain, client_cert, client_key
             )
-            result = Security.SSLSetCertificate(
-                self.context, self._client_cert_chain
-            )
+            result = Security.SSLSetCertificate(self.context, self._client_cert_chain)
             _assert_no_error(result)
 
         while True:
@@ -538,7 +566,7 @@ class WrappedSocket(object):
         # There are some result codes that we want to treat as "not always
         # errors". Specifically, those are errSSLWouldBlock,
         # errSSLClosedGraceful, and errSSLClosedNoNotify.
-        if (result == SecurityConst.errSSLWouldBlock):
+        if result == SecurityConst.errSSLWouldBlock:
             # If we didn't process any bytes, then this was just a time out.
             # However, we can get errSSLWouldBlock in situations when we *did*
             # read some data, and in those cases we should just read "short"
@@ -546,7 +574,10 @@ class WrappedSocket(object):
             if processed_bytes.value == 0:
                 # Timed out, no data read.
                 raise socket.timeout("recv timed out")
-        elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify):
+        elif result in (
+            SecurityConst.errSSLClosedGraceful,
+            SecurityConst.errSSLClosedNoNotify,
+        ):
             # The remote peer has closed this connection. We should do so as
             # well. Note that we don't actually return here because in
             # principle this could actually be fired along with return data.
@@ -585,7 +616,7 @@ class WrappedSocket(object):
     def sendall(self, data):
         total_sent = 0
         while total_sent < len(data):
-            sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
+            sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE])
             total_sent += sent
 
     def shutdown(self):
@@ -632,18 +663,14 @@ class WrappedSocket(object):
         # instead to just flag to urllib3 that it shouldn't do its own hostname
         # validation when using SecureTransport.
         if not binary_form:
-            raise ValueError(
-                "SecureTransport only supports dumping binary certs"
-            )
+            raise ValueError("SecureTransport only supports dumping binary certs")
         trust = Security.SecTrustRef()
         certdata = None
         der_bytes = None
 
         try:
             # Grab the trust store.
-            result = Security.SSLCopyPeerTrust(
-                self.context, ctypes.byref(trust)
-            )
+            result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust))
             _assert_no_error(result)
             if not trust:
                 # Probably we haven't done the handshake yet. No biggie.
@@ -673,6 +700,27 @@ class WrappedSocket(object):
 
         return der_bytes
 
+    def version(self):
+        protocol = Security.SSLProtocol()
+        result = Security.SSLGetNegotiatedProtocolVersion(
+            self.context, ctypes.byref(protocol)
+        )
+        _assert_no_error(result)
+        if protocol.value == SecurityConst.kTLSProtocol13:
+            return "TLSv1.3"
+        elif protocol.value == SecurityConst.kTLSProtocol12:
+            return "TLSv1.2"
+        elif protocol.value == SecurityConst.kTLSProtocol11:
+            return "TLSv1.1"
+        elif protocol.value == SecurityConst.kTLSProtocol1:
+            return "TLSv1"
+        elif protocol.value == SecurityConst.kSSLProtocol3:
+            return "SSLv3"
+        elif protocol.value == SecurityConst.kSSLProtocol2:
+            return "SSLv2"
+        else:
+            raise ssl.SSLError("Unknown TLS version: %r" % protocol)
+
     def _reuse(self):
         self._makefile_refs += 1
 
@@ -684,16 +732,21 @@ class WrappedSocket(object):
 
 
 if _fileobject:  # Platform-specific: Python 2
+
     def makefile(self, mode, bufsize=-1):
         self._makefile_refs += 1
         return _fileobject(self, mode, bufsize, close=True)
+
+
 else:  # Platform-specific: Python 3
+
     def makefile(self, mode="r", buffering=None, *args, **kwargs):
         # We disable buffering with SecureTransport because it conflicts with
         # the buffering that ST does internally (see issue #1153 for more).
         buffering = 0
         return backport_makefile(self, mode, buffering, *args, **kwargs)
 
+
 WrappedSocket.makefile = makefile
 
 
@@ -703,6 +756,7 @@ class SecureTransportContext(object):
     interface of the standard library ``SSLContext`` object to calls into
     SecureTransport.
     """
+
     def __init__(self, protocol):
         self._min_version, self._max_version = _protocol_to_min_max[protocol]
         self._options = 0
@@ -769,16 +823,12 @@ class SecureTransportContext(object):
     def set_ciphers(self, ciphers):
         # For now, we just require the default cipher string.
         if ciphers != util.ssl_.DEFAULT_CIPHERS:
-            raise ValueError(
-                "SecureTransport doesn't support custom cipher strings"
-            )
+            raise ValueError("SecureTransport doesn't support custom cipher strings")
 
     def load_verify_locations(self, cafile=None, capath=None, cadata=None):
         # OK, we only really support cadata and cafile.
         if capath is not None:
-            raise ValueError(
-                "SecureTransport does not support cert directories"
-            )
+            raise ValueError("SecureTransport does not support cert directories")
 
         self._trust_bundle = cafile or cadata
 
@@ -787,9 +837,14 @@ class SecureTransportContext(object):
         self._client_key = keyfile
         self._client_cert_passphrase = password
 
-    def wrap_socket(self, sock, server_side=False,
-                    do_handshake_on_connect=True, suppress_ragged_eofs=True,
-                    server_hostname=None):
+    def wrap_socket(
+        self,
+        sock,
+        server_side=False,
+        do_handshake_on_connect=True,
+        suppress_ragged_eofs=True,
+        server_hostname=None,
+    ):
         # So, what do we do here? Firstly, we assert some properties. This is a
         # stripped down shim, so there is some functionality we don't support.
         # See PEP 543 for the real deal.
@@ -803,8 +858,13 @@ class SecureTransportContext(object):
 
         # Now we can handshake
         wrapped_socket.handshake(
-            server_hostname, self._verify, self._trust_bundle,
-            self._min_version, self._max_version, self._client_cert,
-            self._client_key, self._client_key_passphrase
+            server_hostname,
+            self._verify,
+            self._trust_bundle,
+            self._min_version,
+            self._max_version,
+            self._client_cert,
+            self._client_key,
+            self._client_key_passphrase,
         )
         return wrapped_socket
diff --git a/lib/urllib3/contrib/socks.py b/lib/urllib3/contrib/socks.py
index 39e92fde..9e97f7aa 100644
--- a/lib/urllib3/contrib/socks.py
+++ b/lib/urllib3/contrib/socks.py
@@ -1,25 +1,38 @@
 # -*- coding: utf-8 -*-
 """
 This module contains provisional support for SOCKS proxies from within
-urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and
+urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
 SOCKS5. To enable its functionality, either install PySocks or install this
 module with the ``socks`` extra.
 
 The SOCKS implementation supports the full range of urllib3 features. It also
 supports the following SOCKS features:
 
-- SOCKS4
-- SOCKS4a
-- SOCKS5
+- SOCKS4A (``proxy_url='socks4a://...``)
+- SOCKS4 (``proxy_url='socks4://...``)
+- SOCKS5 with remote DNS (``proxy_url='socks5h://...``)
+- SOCKS5 with local DNS (``proxy_url='socks5://...``)
 - Usernames and passwords for the SOCKS proxy
 
-Known Limitations:
+ .. note::
+    It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in
+    your ``proxy_url`` to ensure that DNS resolution is done from the remote
+    server instead of client-side when connecting to a domain name.
+
+SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5
+supports IPv4, IPv6, and domain names.
+
+When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url``
+will be sent as the ``userid`` section of the SOCKS request::
+
+    proxy_url="socks4a://@proxy-host"
+
+When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion
+of the ``proxy_url`` will be sent as the username/password to authenticate
+with the proxy::
+
+    proxy_url="socks5h://:@proxy-host"
 
-- Currently PySocks does not support contacting remote websites via literal
-  IPv6 addresses. Any such connection attempt will fail. You must use a domain
-  name.
-- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any
-  such connection attempt will fail.
 """
 from __future__ import absolute_import
 
@@ -29,23 +42,20 @@ except ImportError:
     import warnings
     from ..exceptions import DependencyWarning
 
-    warnings.warn((
-        'SOCKS support in urllib3 requires the installation of optional '
-        'dependencies: specifically, PySocks.  For more information, see '
-        'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies'
+    warnings.warn(
+        (
+            "SOCKS support in urllib3 requires the installation of optional "
+            "dependencies: specifically, PySocks.  For more information, see "
+            "https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies"
         ),
-        DependencyWarning
+        DependencyWarning,
     )
     raise
 
 from socket import error as SocketError, timeout as SocketTimeout
 
-from ..connection import (
-    HTTPConnection, HTTPSConnection
-)
-from ..connectionpool import (
-    HTTPConnectionPool, HTTPSConnectionPool
-)
+from ..connection import HTTPConnection, HTTPSConnection
+from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
 from ..exceptions import ConnectTimeoutError, NewConnectionError
 from ..poolmanager import PoolManager
 from ..util.url import parse_url
@@ -60,8 +70,9 @@ class SOCKSConnection(HTTPConnection):
     """
     A plain-text HTTP connection that connects via a SOCKS proxy.
     """
+
     def __init__(self, *args, **kwargs):
-        self._socks_options = kwargs.pop('_socks_options')
+        self._socks_options = kwargs.pop("_socks_options")
         super(SOCKSConnection, self).__init__(*args, **kwargs)
 
     def _new_conn(self):
@@ -70,28 +81,30 @@ class SOCKSConnection(HTTPConnection):
         """
         extra_kw = {}
         if self.source_address:
-            extra_kw['source_address'] = self.source_address
+            extra_kw["source_address"] = self.source_address
 
         if self.socket_options:
-            extra_kw['socket_options'] = self.socket_options
+            extra_kw["socket_options"] = self.socket_options
 
         try:
             conn = socks.create_connection(
                 (self.host, self.port),
-                proxy_type=self._socks_options['socks_version'],
-                proxy_addr=self._socks_options['proxy_host'],
-                proxy_port=self._socks_options['proxy_port'],
-                proxy_username=self._socks_options['username'],
-                proxy_password=self._socks_options['password'],
-                proxy_rdns=self._socks_options['rdns'],
+                proxy_type=self._socks_options["socks_version"],
+                proxy_addr=self._socks_options["proxy_host"],
+                proxy_port=self._socks_options["proxy_port"],
+                proxy_username=self._socks_options["username"],
+                proxy_password=self._socks_options["password"],
+                proxy_rdns=self._socks_options["rdns"],
                 timeout=self.timeout,
                 **extra_kw
             )
 
-        except SocketTimeout as e:
+        except SocketTimeout:
             raise ConnectTimeoutError(
-                self, "Connection to %s timed out. (connect timeout=%s)" %
-                (self.host, self.timeout))
+                self,
+                "Connection to %s timed out. (connect timeout=%s)"
+                % (self.host, self.timeout),
+            )
 
         except socks.ProxyError as e:
             # This is fragile as hell, but it seems to be the only way to raise
@@ -101,23 +114,22 @@ class SOCKSConnection(HTTPConnection):
                 if isinstance(error, SocketTimeout):
                     raise ConnectTimeoutError(
                         self,
-                        "Connection to %s timed out. (connect timeout=%s)" %
-                        (self.host, self.timeout)
+                        "Connection to %s timed out. (connect timeout=%s)"
+                        % (self.host, self.timeout),
                     )
                 else:
                     raise NewConnectionError(
-                        self,
-                        "Failed to establish a new connection: %s" % error
+                        self, "Failed to establish a new connection: %s" % error
                     )
             else:
                 raise NewConnectionError(
-                    self,
-                    "Failed to establish a new connection: %s" % e
+                    self, "Failed to establish a new connection: %s" % e
                 )
 
         except SocketError as e:  # Defensive: PySocks should catch all these.
             raise NewConnectionError(
-                self, "Failed to establish a new connection: %s" % e)
+                self, "Failed to establish a new connection: %s" % e
+            )
 
         return conn
 
@@ -143,43 +155,53 @@ class SOCKSProxyManager(PoolManager):
     A version of the urllib3 ProxyManager that routes connections via the
     defined SOCKS proxy.
     """
+
     pool_classes_by_scheme = {
-        'http': SOCKSHTTPConnectionPool,
-        'https': SOCKSHTTPSConnectionPool,
+        "http": SOCKSHTTPConnectionPool,
+        "https": SOCKSHTTPSConnectionPool,
     }
 
-    def __init__(self, proxy_url, username=None, password=None,
-                 num_pools=10, headers=None, **connection_pool_kw):
+    def __init__(
+        self,
+        proxy_url,
+        username=None,
+        password=None,
+        num_pools=10,
+        headers=None,
+        **connection_pool_kw
+    ):
         parsed = parse_url(proxy_url)
 
-        if parsed.scheme == 'socks5':
+        if username is None and password is None and parsed.auth is not None:
+            split = parsed.auth.split(":")
+            if len(split) == 2:
+                username, password = split
+        if parsed.scheme == "socks5":
             socks_version = socks.PROXY_TYPE_SOCKS5
             rdns = False
-        elif parsed.scheme == 'socks5h':
+        elif parsed.scheme == "socks5h":
             socks_version = socks.PROXY_TYPE_SOCKS5
             rdns = True
-        elif parsed.scheme == 'socks4':
+        elif parsed.scheme == "socks4":
             socks_version = socks.PROXY_TYPE_SOCKS4
             rdns = False
-        elif parsed.scheme == 'socks4a':
+        elif parsed.scheme == "socks4a":
             socks_version = socks.PROXY_TYPE_SOCKS4
             rdns = True
         else:
-            raise ValueError(
-                "Unable to determine SOCKS version from %s" % proxy_url
-            )
+            raise ValueError("Unable to determine SOCKS version from %s" % proxy_url)
 
         self.proxy_url = proxy_url
 
         socks_options = {
-            'socks_version': socks_version,
-            'proxy_host': parsed.host,
-            'proxy_port': parsed.port,
-            'username': username,
-            'password': password,
-            'rdns': rdns
+            "socks_version": socks_version,
+            "proxy_host": parsed.host,
+            "proxy_port": parsed.port,
+            "username": username,
+            "password": password,
+            "rdns": rdns,
         }
-        connection_pool_kw['_socks_options'] = socks_options
+        connection_pool_kw["_socks_options"] = socks_options
 
         super(SOCKSProxyManager, self).__init__(
             num_pools, headers, **connection_pool_kw
diff --git a/lib/urllib3/exceptions.py b/lib/urllib3/exceptions.py
index 6c4be581..93d93fba 100644
--- a/lib/urllib3/exceptions.py
+++ b/lib/urllib3/exceptions.py
@@ -1,7 +1,6 @@
 from __future__ import absolute_import
-from .packages.six.moves.http_client import (
-    IncompleteRead as httplib_IncompleteRead
-)
+from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead
+
 # Base Exceptions
 
 
@@ -17,6 +16,7 @@ class HTTPWarning(Warning):
 
 class PoolError(HTTPError):
     "Base exception for errors caused within a pool."
+
     def __init__(self, pool, message):
         self.pool = pool
         HTTPError.__init__(self, "%s: %s" % (pool, message))
@@ -28,6 +28,7 @@ class PoolError(HTTPError):
 
 class RequestError(PoolError):
     "Base exception for PoolErrors that have associated URLs."
+
     def __init__(self, pool, url, message):
         self.url = url
         PoolError.__init__(self, pool, message)
@@ -63,6 +64,7 @@ ConnectionError = ProtocolError
 
 # Leaf Exceptions
 
+
 class MaxRetryError(RequestError):
     """Raised when the maximum number of retries is exceeded.
 
@@ -76,8 +78,7 @@ class MaxRetryError(RequestError):
     def __init__(self, pool, url, reason=None):
         self.reason = reason
 
-        message = "Max retries exceeded with url: %s (Caused by %r)" % (
-            url, reason)
+        message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason)
 
         RequestError.__init__(self, pool, url, message)
 
@@ -93,6 +94,7 @@ class HostChangedError(RequestError):
 
 class TimeoutStateError(HTTPError):
     """ Raised when passing an invalid state to a timeout """
+
     pass
 
 
@@ -102,6 +104,7 @@ class TimeoutError(HTTPError):
     Catching this error will catch both :exc:`ReadTimeoutErrors
     ` and :exc:`ConnectTimeoutErrors `.
     """
+
     pass
 
 
@@ -149,12 +152,12 @@ class LocationParseError(LocationValueError):
 
 class ResponseError(HTTPError):
     "Used as a container for an error reason supplied in a MaxRetryError."
-    GENERIC_ERROR = 'too many error responses'
-    SPECIFIC_ERROR = 'too many {status_code} error responses'
+    GENERIC_ERROR = "too many error responses"
+    SPECIFIC_ERROR = "too many {status_code} error responses"
 
 
 class SecurityWarning(HTTPWarning):
-    "Warned when perfoming security reducing actions"
+    "Warned when performing security reducing actions"
     pass
 
 
@@ -188,6 +191,7 @@ class DependencyWarning(HTTPWarning):
     Warned when an attempt is made to import a module with missing optional
     dependencies.
     """
+
     pass
 
 
@@ -201,6 +205,7 @@ class BodyNotHttplibCompatible(HTTPError):
     Body should be httplib.HTTPResponse like (have an fp attribute which
     returns raw chunks) for read_chunked().
     """
+
     pass
 
 
@@ -212,12 +217,15 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
     for `partial` to avoid creating large objects on streamed
     reads.
     """
+
     def __init__(self, partial, expected):
         super(IncompleteRead, self).__init__(partial, expected)
 
     def __repr__(self):
-        return ('IncompleteRead(%i bytes read, '
-                '%i more expected)' % (self.partial, self.expected))
+        return "IncompleteRead(%i bytes read, " "%i more expected)" % (
+            self.partial,
+            self.expected,
+        )
 
 
 class InvalidHeader(HTTPError):
@@ -236,8 +244,9 @@ class ProxySchemeUnknown(AssertionError, ValueError):
 
 class HeaderParsingError(HTTPError):
     "Raised by assert_header_parsing, but we convert it to a log.warning statement."
+
     def __init__(self, defects, unparsed_data):
-        message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data)
+        message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data)
         super(HeaderParsingError, self).__init__(message)
 
 
diff --git a/lib/urllib3/fields.py b/lib/urllib3/fields.py
index 19b0ae0c..8715b220 100644
--- a/lib/urllib3/fields.py
+++ b/lib/urllib3/fields.py
@@ -1,11 +1,12 @@
 from __future__ import absolute_import
 import email.utils
 import mimetypes
+import re
 
 from .packages import six
 
 
-def guess_content_type(filename, default='application/octet-stream'):
+def guess_content_type(filename, default="application/octet-stream"):
     """
     Guess the "Content-Type" of a file.
 
@@ -19,57 +20,143 @@ def guess_content_type(filename, default='application/octet-stream'):
     return default
 
 
-def format_header_param(name, value):
+def format_header_param_rfc2231(name, value):
     """
-    Helper function to format and quote a single header parameter.
+    Helper function to format and quote a single header parameter using the
+    strategy defined in RFC 2231.
 
     Particularly useful for header parameters which might contain
-    non-ASCII values, like file names. This follows RFC 2231, as
-    suggested by RFC 2388 Section 4.4.
+    non-ASCII values, like file names. This follows RFC 2388 Section 4.4.
 
     :param name:
         The name of the parameter, a string expected to be ASCII only.
     :param value:
-        The value of the parameter, provided as a unicode string.
+        The value of the parameter, provided as ``bytes`` or `str``.
+    :ret:
+        An RFC-2231-formatted unicode string.
     """
+    if isinstance(value, six.binary_type):
+        value = value.decode("utf-8")
+
     if not any(ch in value for ch in '"\\\r\n'):
-        result = '%s="%s"' % (name, value)
+        result = u'%s="%s"' % (name, value)
         try:
-            result.encode('ascii')
+            result.encode("ascii")
         except (UnicodeEncodeError, UnicodeDecodeError):
             pass
         else:
             return result
-    if not six.PY3 and isinstance(value, six.text_type):  # Python 2:
-        value = value.encode('utf-8')
-    value = email.utils.encode_rfc2231(value, 'utf-8')
-    value = '%s*=%s' % (name, value)
+
+    if six.PY2:  # Python 2:
+        value = value.encode("utf-8")
+
+    # encode_rfc2231 accepts an encoded string and returns an ascii-encoded
+    # string in Python 2 but accepts and returns unicode strings in Python 3
+    value = email.utils.encode_rfc2231(value, "utf-8")
+    value = "%s*=%s" % (name, value)
+
+    if six.PY2:  # Python 2:
+        value = value.decode("utf-8")
+
     return value
 
 
+_HTML5_REPLACEMENTS = {
+    u"\u0022": u"%22",
+    # Replace "\" with "\\".
+    u"\u005C": u"\u005C\u005C",
+    u"\u005C": u"\u005C\u005C",
+}
+
+# All control characters from 0x00 to 0x1F *except* 0x1B.
+_HTML5_REPLACEMENTS.update(
+    {
+        six.unichr(cc): u"%{:02X}".format(cc)
+        for cc in range(0x00, 0x1F + 1)
+        if cc not in (0x1B,)
+    }
+)
+
+
+def _replace_multiple(value, needles_and_replacements):
+    def replacer(match):
+        return needles_and_replacements[match.group(0)]
+
+    pattern = re.compile(
+        r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()])
+    )
+
+    result = pattern.sub(replacer, value)
+
+    return result
+
+
+def format_header_param_html5(name, value):
+    """
+    Helper function to format and quote a single header parameter using the
+    HTML5 strategy.
+
+    Particularly useful for header parameters which might contain
+    non-ASCII values, like file names. This follows the `HTML5 Working Draft
+    Section 4.10.22.7`_ and matches the behavior of curl and modern browsers.
+
+    .. _HTML5 Working Draft Section 4.10.22.7:
+        https://w3c.github.io/html/sec-forms.html#multipart-form-data
+
+    :param name:
+        The name of the parameter, a string expected to be ASCII only.
+    :param value:
+        The value of the parameter, provided as ``bytes`` or `str``.
+    :ret:
+        A unicode string, stripped of troublesome characters.
+    """
+    if isinstance(value, six.binary_type):
+        value = value.decode("utf-8")
+
+    value = _replace_multiple(value, _HTML5_REPLACEMENTS)
+
+    return u'%s="%s"' % (name, value)
+
+
+# For backwards-compatibility.
+format_header_param = format_header_param_html5
+
+
 class RequestField(object):
     """
     A data container for request body parameters.
 
     :param name:
-        The name of this request field.
+        The name of this request field. Must be unicode.
     :param data:
         The data/value body.
     :param filename:
-        An optional filename of the request field.
+        An optional filename of the request field. Must be unicode.
     :param headers:
         An optional dict-like object of headers to initially use for the field.
+    :param header_formatter:
+        An optional callable that is used to encode and format the headers. By
+        default, this is :func:`format_header_param_html5`.
     """
-    def __init__(self, name, data, filename=None, headers=None):
+
+    def __init__(
+        self,
+        name,
+        data,
+        filename=None,
+        headers=None,
+        header_formatter=format_header_param_html5,
+    ):
         self._name = name
         self._filename = filename
         self.data = data
         self.headers = {}
         if headers:
             self.headers = dict(headers)
+        self.header_formatter = header_formatter
 
     @classmethod
-    def from_tuples(cls, fieldname, value):
+    def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5):
         """
         A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
 
@@ -97,21 +184,25 @@ class RequestField(object):
             content_type = None
             data = value
 
-        request_param = cls(fieldname, data, filename=filename)
+        request_param = cls(
+            fieldname, data, filename=filename, header_formatter=header_formatter
+        )
         request_param.make_multipart(content_type=content_type)
 
         return request_param
 
     def _render_part(self, name, value):
         """
-        Overridable helper function to format a single header parameter.
+        Overridable helper function to format a single header parameter. By
+        default, this calls ``self.header_formatter``.
 
         :param name:
             The name of the parameter, a string expected to be ASCII only.
         :param value:
             The value of the parameter, provided as a unicode string.
         """
-        return format_header_param(name, value)
+
+        return self.header_formatter(name, value)
 
     def _render_parts(self, header_parts):
         """
@@ -121,7 +212,7 @@ class RequestField(object):
         'Content-Disposition' fields.
 
         :param header_parts:
-            A sequence of (k, v) typles or a :class:`dict` of (k, v) to format
+            A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
             as `k1="v1"; k2="v2"; ...`.
         """
         parts = []
@@ -133,7 +224,7 @@ class RequestField(object):
             if value is not None:
                 parts.append(self._render_part(name, value))
 
-        return '; '.join(parts)
+        return u"; ".join(parts)
 
     def render_headers(self):
         """
@@ -141,21 +232,22 @@ class RequestField(object):
         """
         lines = []
 
-        sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location']
+        sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
         for sort_key in sort_keys:
             if self.headers.get(sort_key, False):
-                lines.append('%s: %s' % (sort_key, self.headers[sort_key]))
+                lines.append(u"%s: %s" % (sort_key, self.headers[sort_key]))
 
         for header_name, header_value in self.headers.items():
             if header_name not in sort_keys:
                 if header_value:
-                    lines.append('%s: %s' % (header_name, header_value))
+                    lines.append(u"%s: %s" % (header_name, header_value))
 
-        lines.append('\r\n')
-        return '\r\n'.join(lines)
+        lines.append(u"\r\n")
+        return u"\r\n".join(lines)
 
-    def make_multipart(self, content_disposition=None, content_type=None,
-                       content_location=None):
+    def make_multipart(
+        self, content_disposition=None, content_type=None, content_location=None
+    ):
         """
         Makes this request field into a multipart request field.
 
@@ -168,11 +260,14 @@ class RequestField(object):
             The 'Content-Location' of the request body.
 
         """
-        self.headers['Content-Disposition'] = content_disposition or 'form-data'
-        self.headers['Content-Disposition'] += '; '.join([
-            '', self._render_parts(
-                (('name', self._name), ('filename', self._filename))
-            )
-        ])
-        self.headers['Content-Type'] = content_type
-        self.headers['Content-Location'] = content_location
+        self.headers["Content-Disposition"] = content_disposition or u"form-data"
+        self.headers["Content-Disposition"] += u"; ".join(
+            [
+                u"",
+                self._render_parts(
+                    ((u"name", self._name), (u"filename", self._filename))
+                ),
+            ]
+        )
+        self.headers["Content-Type"] = content_type
+        self.headers["Content-Location"] = content_location
diff --git a/lib/urllib3/filepost.py b/lib/urllib3/filepost.py
index cd11cee4..b7b00992 100644
--- a/lib/urllib3/filepost.py
+++ b/lib/urllib3/filepost.py
@@ -1,21 +1,25 @@
 from __future__ import absolute_import
+import binascii
 import codecs
+import os
 
-from uuid import uuid4
 from io import BytesIO
 
 from .packages import six
 from .packages.six import b
 from .fields import RequestField
 
-writer = codecs.lookup('utf-8')[3]
+writer = codecs.lookup("utf-8")[3]
 
 
 def choose_boundary():
     """
     Our embarrassingly-simple replacement for mimetools.choose_boundary.
     """
-    return uuid4().hex
+    boundary = binascii.hexlify(os.urandom(16))
+    if not six.PY2:
+        boundary = boundary.decode("ascii")
+    return boundary
 
 
 def iter_field_objects(fields):
@@ -65,14 +69,14 @@ def encode_multipart_formdata(fields, boundary=None):
 
     :param boundary:
         If not specified, then a random boundary will be generated using
-        :func:`mimetools.choose_boundary`.
+        :func:`urllib3.filepost.choose_boundary`.
     """
     body = BytesIO()
     if boundary is None:
         boundary = choose_boundary()
 
     for field in iter_field_objects(fields):
-        body.write(b('--%s\r\n' % (boundary)))
+        body.write(b("--%s\r\n" % (boundary)))
 
         writer(body).write(field.render_headers())
         data = field.data
@@ -85,10 +89,10 @@ def encode_multipart_formdata(fields, boundary=None):
         else:
             body.write(data)
 
-        body.write(b'\r\n')
+        body.write(b"\r\n")
 
-    body.write(b('--%s--\r\n' % (boundary)))
+    body.write(b("--%s--\r\n" % (boundary)))
 
-    content_type = str('multipart/form-data; boundary=%s' % boundary)
+    content_type = str("multipart/form-data; boundary=%s" % boundary)
 
     return body.getvalue(), content_type
diff --git a/lib/urllib3/packages/__init__.py b/lib/urllib3/packages/__init__.py
index 170e974c..fce4caa6 100644
--- a/lib/urllib3/packages/__init__.py
+++ b/lib/urllib3/packages/__init__.py
@@ -2,4 +2,4 @@ from __future__ import absolute_import
 
 from . import ssl_match_hostname
 
-__all__ = ('ssl_match_hostname', )
+__all__ = ("ssl_match_hostname",)
diff --git a/lib/urllib3/packages/backports/__init__.py b/lib/urllib3/packages/backports/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lib/urllib3/packages/backports/makefile.py b/lib/urllib3/packages/backports/makefile.py
index 75b80dcf..a3156a69 100644
--- a/lib/urllib3/packages/backports/makefile.py
+++ b/lib/urllib3/packages/backports/makefile.py
@@ -11,15 +11,14 @@ import io
 from socket import SocketIO
 
 
-def backport_makefile(self, mode="r", buffering=None, encoding=None,
-                      errors=None, newline=None):
+def backport_makefile(
+    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
+):
     """
     Backport of ``socket.makefile`` from Python 3.5.
     """
-    if not set(mode) <= set(["r", "w", "b"]):
-        raise ValueError(
-            "invalid mode %r (only r, w, b allowed)" % (mode,)
-        )
+    if not set(mode) <= {"r", "w", "b"}:
+        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
     writing = "w" in mode
     reading = "r" in mode or not writing
     assert reading or writing
diff --git a/lib/urllib3/packages/ordered_dict.py b/lib/urllib3/packages/ordered_dict.py
deleted file mode 100644
index 4479363c..00000000
--- a/lib/urllib3/packages/ordered_dict.py
+++ /dev/null
@@ -1,259 +0,0 @@
-# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
-# Passes Python2.7's test suite and incorporates all the latest updates.
-# Copyright 2009 Raymond Hettinger, released under the MIT License.
-# http://code.activestate.com/recipes/576693/
-try:
-    from thread import get_ident as _get_ident
-except ImportError:
-    from dummy_thread import get_ident as _get_ident
-
-try:
-    from _abcoll import KeysView, ValuesView, ItemsView
-except ImportError:
-    pass
-
-
-class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as for regular dictionaries.
-
-    # The internal self.__map dictionary maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
-
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  Signature is the same as for
-        regular dictionaries, but keyword arguments are not recommended
-        because their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link which goes at the end of the linked
-        # list, and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which is
-        # then removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, key = self.__map.pop(key)
-        link_prev[1] = link_next
-        link_next[0] = link_prev
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        root = self.__root
-        curr = root[1]
-        while curr is not root:
-            yield curr[2]
-            curr = curr[1]
-
-    def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        root = self.__root
-        curr = root[0]
-        while curr is not root:
-            yield curr[2]
-            curr = curr[0]
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        try:
-            for node in self.__map.itervalues():
-                del node[:]
-            root = self.__root
-            root[:] = [root, root, None]
-            self.__map.clear()
-        except AttributeError:
-            pass
-        dict.clear(self)
-
-    def popitem(self, last=True):
-        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
-        Pairs are returned in LIFO order if last is true or FIFO order if false.
-
-        '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        root = self.__root
-        if last:
-            link = root[0]
-            link_prev = link[0]
-            link_prev[1] = root
-            root[0] = link_prev
-        else:
-            link = root[1]
-            link_next = link[1]
-            root[1] = link_next
-            link_next[0] = root
-        key = link[2]
-        del self.__map[key]
-        value = dict.pop(self, key)
-        return key, value
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) items in od'
-        for k in self:
-            yield (k, self[k])
-
-    def update(*args, **kwds):
-        '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.
-
-        If E is a dict instance, does:           for k in E: od[k] = E[k]
-        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
-        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
-        In either case, this is followed by:     for k, v in F.items(): od[k] = v
-
-        '''
-        if len(args) > 2:
-            raise TypeError('update() takes at most 2 positional '
-                            'arguments (%d given)' % (len(args),))
-        elif not args:
-            raise TypeError('update() takes at least 1 argument (0 given)')
-        self = args[0]
-        # Make progressively weaker assumptions about "other"
-        other = ()
-        if len(args) == 2:
-            other = args[1]
-        if isinstance(other, dict):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, 'keys'):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
-        for key, value in kwds.items():
-            self[key] = value
-
-    __update = update  # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
-        If key is not found, d is returned if given, otherwise KeyError is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
-
-    def __repr__(self, _repr_running={}):
-        'od.__repr__() <==> repr(od)'
-        call_key = id(self), _get_ident()
-        if call_key in _repr_running:
-            return '...'
-        _repr_running[call_key] = 1
-        try:
-            if not self:
-                return '%s()' % (self.__class__.__name__,)
-            return '%s(%r)' % (self.__class__.__name__, self.items())
-        finally:
-            del _repr_running[call_key]
-
-    def __reduce__(self):
-        'Return state information for pickling'
-        items = [[k, self[k]] for k in self]
-        inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
-        if inst_dict:
-            return (self.__class__, (items,), inst_dict)
-        return self.__class__, (items,)
-
-    def copy(self):
-        'od.copy() -> a shallow copy of od'
-        return self.__class__(self)
-
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
-        and values equal to v (which defaults to None).
-
-        '''
-        d = cls()
-        for key in iterable:
-            d[key] = value
-        return d
-
-    def __eq__(self, other):
-        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
-        while comparison to a regular mapping is order-insensitive.
-
-        '''
-        if isinstance(other, OrderedDict):
-            return len(self)==len(other) and self.items() == other.items()
-        return dict.__eq__(self, other)
-
-    def __ne__(self, other):
-        return not self == other
-
-    # -- the following methods are only used in Python 2.7 --
-
-    def viewkeys(self):
-        "od.viewkeys() -> a set-like object providing a view on od's keys"
-        return KeysView(self)
-
-    def viewvalues(self):
-        "od.viewvalues() -> an object providing a view on od's values"
-        return ValuesView(self)
-
-    def viewitems(self):
-        "od.viewitems() -> a set-like object providing a view on od's items"
-        return ItemsView(self)
diff --git a/lib/urllib3/packages/six.py b/lib/urllib3/packages/six.py
index 190c0239..31442409 100644
--- a/lib/urllib3/packages/six.py
+++ b/lib/urllib3/packages/six.py
@@ -1,6 +1,4 @@
-"""Utilities for writing code that runs on Python 2 and 3"""
-
-# Copyright (c) 2010-2015 Benjamin Peterson
+# Copyright (c) 2010-2019 Benjamin Peterson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -20,6 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+"""Utilities for writing code that runs on Python 2 and 3"""
+
 from __future__ import absolute_import
 
 import functools
@@ -29,7 +29,7 @@ import sys
 import types
 
 __author__ = "Benjamin Peterson "
-__version__ = "1.10.0"
+__version__ = "1.12.0"
 
 
 # Useful for very coarse version differentiation.
@@ -38,15 +38,15 @@ PY3 = sys.version_info[0] == 3
 PY34 = sys.version_info[0:2] >= (3, 4)
 
 if PY3:
-    string_types = str,
-    integer_types = int,
-    class_types = type,
+    string_types = (str,)
+    integer_types = (int,)
+    class_types = (type,)
     text_type = str
     binary_type = bytes
 
     MAXSIZE = sys.maxsize
 else:
-    string_types = basestring,
+    string_types = (basestring,)
     integer_types = (int, long)
     class_types = (type, types.ClassType)
     text_type = unicode
@@ -58,9 +58,9 @@ else:
     else:
         # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
         class X(object):
-
             def __len__(self):
                 return 1 << 31
+
         try:
             len(X())
         except OverflowError:
@@ -84,7 +84,6 @@ def _import_module(name):
 
 
 class _LazyDescr(object):
-
     def __init__(self, name):
         self.name = name
 
@@ -101,7 +100,6 @@ class _LazyDescr(object):
 
 
 class MovedModule(_LazyDescr):
-
     def __init__(self, name, old, new=None):
         super(MovedModule, self).__init__(name)
         if PY3:
@@ -122,7 +120,6 @@ class MovedModule(_LazyDescr):
 
 
 class _LazyModule(types.ModuleType):
-
     def __init__(self, name):
         super(_LazyModule, self).__init__(name)
         self.__doc__ = self.__class__.__doc__
@@ -137,7 +134,6 @@ class _LazyModule(types.ModuleType):
 
 
 class MovedAttribute(_LazyDescr):
-
     def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
         super(MovedAttribute, self).__init__(name)
         if PY3:
@@ -221,28 +217,36 @@ class _SixMetaPathImporter(object):
         Required, if is_package is implemented"""
         self.__get_module(fullname)  # eventually raises ImportError
         return None
+
     get_source = get_code  # same as get_code
 
+
 _importer = _SixMetaPathImporter(__name__)
 
 
 class _MovedItems(_LazyModule):
 
     """Lazy loading of moved objects"""
+
     __path__ = []  # mark as package
 
 
 _moved_attributes = [
     MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
     MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
-    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute(
+        "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"
+    ),
     MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
     MovedAttribute("intern", "__builtin__", "sys"),
     MovedAttribute("map", "itertools", "builtins", "imap", "map"),
     MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
     MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+    MovedAttribute("getoutput", "commands", "subprocess"),
     MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
-    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+    MovedAttribute(
+        "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"
+    ),
     MovedAttribute("reduce", "__builtin__", "functools"),
     MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
     MovedAttribute("StringIO", "StringIO", "io"),
@@ -251,7 +255,9 @@ _moved_attributes = [
     MovedAttribute("UserString", "UserString", "collections"),
     MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
     MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
-    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+    MovedAttribute(
+        "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"
+    ),
     MovedModule("builtins", "__builtin__"),
     MovedModule("configparser", "ConfigParser"),
     MovedModule("copyreg", "copy_reg"),
@@ -262,10 +268,13 @@ _moved_attributes = [
     MovedModule("html_entities", "htmlentitydefs", "html.entities"),
     MovedModule("html_parser", "HTMLParser", "html.parser"),
     MovedModule("http_client", "httplib", "http.client"),
-    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
-    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
-    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
     MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule(
+        "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"
+    ),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
     MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
     MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
     MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
@@ -283,15 +292,12 @@ _moved_attributes = [
     MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
     MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
     MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
-    MovedModule("tkinter_colorchooser", "tkColorChooser",
-                "tkinter.colorchooser"),
-    MovedModule("tkinter_commondialog", "tkCommonDialog",
-                "tkinter.commondialog"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"),
     MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
     MovedModule("tkinter_font", "tkFont", "tkinter.font"),
     MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
-    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
-                "tkinter.simpledialog"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"),
     MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
     MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
     MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
@@ -301,9 +307,7 @@ _moved_attributes = [
 ]
 # Add windows specific modules.
 if sys.platform == "win32":
-    _moved_attributes += [
-        MovedModule("winreg", "_winreg"),
-    ]
+    _moved_attributes += [MovedModule("winreg", "_winreg")]
 
 for attr in _moved_attributes:
     setattr(_MovedItems, attr.name, attr)
@@ -337,10 +341,14 @@ _urllib_parse_moved_attributes = [
     MovedAttribute("quote_plus", "urllib", "urllib.parse"),
     MovedAttribute("unquote", "urllib", "urllib.parse"),
     MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute(
+        "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"
+    ),
     MovedAttribute("urlencode", "urllib", "urllib.parse"),
     MovedAttribute("splitquery", "urllib", "urllib.parse"),
     MovedAttribute("splittag", "urllib", "urllib.parse"),
     MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("splitvalue", "urllib", "urllib.parse"),
     MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
     MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
     MovedAttribute("uses_params", "urlparse", "urllib.parse"),
@@ -353,8 +361,11 @@ del attr
 
 Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
 
-_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
-                      "moves.urllib_parse", "moves.urllib.parse")
+_importer._add_module(
+    Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+    "moves.urllib_parse",
+    "moves.urllib.parse",
+)
 
 
 class Module_six_moves_urllib_error(_LazyModule):
@@ -373,8 +384,11 @@ del attr
 
 Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
 
-_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
-                      "moves.urllib_error", "moves.urllib.error")
+_importer._add_module(
+    Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+    "moves.urllib_error",
+    "moves.urllib.error",
+)
 
 
 class Module_six_moves_urllib_request(_LazyModule):
@@ -416,6 +430,8 @@ _urllib_request_moved_attributes = [
     MovedAttribute("URLopener", "urllib", "urllib.request"),
     MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
     MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+    MovedAttribute("parse_http_list", "urllib2", "urllib.request"),
+    MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"),
 ]
 for attr in _urllib_request_moved_attributes:
     setattr(Module_six_moves_urllib_request, attr.name, attr)
@@ -423,8 +439,11 @@ del attr
 
 Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
 
-_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
-                      "moves.urllib_request", "moves.urllib.request")
+_importer._add_module(
+    Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+    "moves.urllib_request",
+    "moves.urllib.request",
+)
 
 
 class Module_six_moves_urllib_response(_LazyModule):
@@ -444,8 +463,11 @@ del attr
 
 Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
 
-_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
-                      "moves.urllib_response", "moves.urllib.response")
+_importer._add_module(
+    Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+    "moves.urllib_response",
+    "moves.urllib.response",
+)
 
 
 class Module_six_moves_urllib_robotparser(_LazyModule):
@@ -454,21 +476,27 @@ class Module_six_moves_urllib_robotparser(_LazyModule):
 
 
 _urllib_robotparser_moved_attributes = [
-    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser")
 ]
 for attr in _urllib_robotparser_moved_attributes:
     setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
 del attr
 
-Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+Module_six_moves_urllib_robotparser._moved_attributes = (
+    _urllib_robotparser_moved_attributes
+)
 
-_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
-                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+_importer._add_module(
+    Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+    "moves.urllib_robotparser",
+    "moves.urllib.robotparser",
+)
 
 
 class Module_six_moves_urllib(types.ModuleType):
 
     """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+
     __path__ = []  # mark as package
     parse = _importer._get_module("moves.urllib_parse")
     error = _importer._get_module("moves.urllib_error")
@@ -477,10 +505,12 @@ class Module_six_moves_urllib(types.ModuleType):
     robotparser = _importer._get_module("moves.urllib_robotparser")
 
     def __dir__(self):
-        return ['parse', 'error', 'request', 'response', 'robotparser']
+        return ["parse", "error", "request", "response", "robotparser"]
 
-_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
-                      "moves.urllib")
+
+_importer._add_module(
+    Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib"
+)
 
 
 def add_move(move):
@@ -520,19 +550,24 @@ else:
 try:
     advance_iterator = next
 except NameError:
+
     def advance_iterator(it):
         return it.next()
+
+
 next = advance_iterator
 
 
 try:
     callable = callable
 except NameError:
+
     def callable(obj):
         return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
 
 
 if PY3:
+
     def get_unbound_function(unbound):
         return unbound
 
@@ -543,6 +578,7 @@ if PY3:
 
     Iterator = object
 else:
+
     def get_unbound_function(unbound):
         return unbound.im_func
 
@@ -553,13 +589,13 @@ else:
         return types.MethodType(func, None, cls)
 
     class Iterator(object):
-
         def next(self):
             return type(self).__next__(self)
 
     callable = callable
-_add_doc(get_unbound_function,
-         """Get the function out of a possibly unbound function""")
+_add_doc(
+    get_unbound_function, """Get the function out of a possibly unbound function"""
+)
 
 
 get_method_function = operator.attrgetter(_meth_func)
@@ -571,6 +607,7 @@ get_function_globals = operator.attrgetter(_func_globals)
 
 
 if PY3:
+
     def iterkeys(d, **kw):
         return iter(d.keys(**kw))
 
@@ -589,6 +626,7 @@ if PY3:
 
     viewitems = operator.methodcaller("items")
 else:
+
     def iterkeys(d, **kw):
         return d.iterkeys(**kw)
 
@@ -609,28 +647,33 @@ else:
 
 _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
 _add_doc(itervalues, "Return an iterator over the values of a dictionary.")
-_add_doc(iteritems,
-         "Return an iterator over the (key, value) pairs of a dictionary.")
-_add_doc(iterlists,
-         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(
+    iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary."
+)
 
 
 if PY3:
+
     def b(s):
         return s.encode("latin-1")
 
     def u(s):
         return s
+
     unichr = chr
     import struct
+
     int2byte = struct.Struct(">B").pack
     del struct
     byte2int = operator.itemgetter(0)
     indexbytes = operator.getitem
     iterbytes = iter
     import io
+
     StringIO = io.StringIO
     BytesIO = io.BytesIO
+    del io
     _assertCountEqual = "assertCountEqual"
     if sys.version_info[1] <= 1:
         _assertRaisesRegex = "assertRaisesRegexp"
@@ -639,12 +682,15 @@ if PY3:
         _assertRaisesRegex = "assertRaisesRegex"
         _assertRegex = "assertRegex"
 else:
+
     def b(s):
         return s
+
     # Workaround for standalone backslash
 
     def u(s):
-        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+        return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape")
+
     unichr = unichr
     int2byte = chr
 
@@ -653,8 +699,10 @@ else:
 
     def indexbytes(buf, i):
         return ord(buf[i])
+
     iterbytes = functools.partial(itertools.imap, ord)
     import StringIO
+
     StringIO = BytesIO = StringIO.StringIO
     _assertCountEqual = "assertItemsEqual"
     _assertRaisesRegex = "assertRaisesRegexp"
@@ -679,13 +727,19 @@ if PY3:
     exec_ = getattr(moves.builtins, "exec")
 
     def reraise(tp, value, tb=None):
-        if value is None:
-            value = tp()
-        if value.__traceback__ is not tb:
-            raise value.with_traceback(tb)
-        raise value
+        try:
+            if value is None:
+                value = tp()
+            if value.__traceback__ is not tb:
+                raise value.with_traceback(tb)
+            raise value
+        finally:
+            value = None
+            tb = None
+
 
 else:
+
     def exec_(_code_, _globs_=None, _locs_=None):
         """Execute code in a namespace."""
         if _globs_ is None:
@@ -698,28 +752,45 @@ else:
             _locs_ = _globs_
         exec("""exec _code_ in _globs_, _locs_""")
 
-    exec_("""def reraise(tp, value, tb=None):
-    raise tp, value, tb
-""")
+    exec_(
+        """def reraise(tp, value, tb=None):
+    try:
+        raise tp, value, tb
+    finally:
+        tb = None
+"""
+    )
 
 
 if sys.version_info[:2] == (3, 2):
-    exec_("""def raise_from(value, from_value):
-    if from_value is None:
-        raise value
-    raise value from from_value
-""")
+    exec_(
+        """def raise_from(value, from_value):
+    try:
+        if from_value is None:
+            raise value
+        raise value from from_value
+    finally:
+        value = None
+"""
+    )
 elif sys.version_info[:2] > (3, 2):
-    exec_("""def raise_from(value, from_value):
-    raise value from from_value
-""")
+    exec_(
+        """def raise_from(value, from_value):
+    try:
+        raise value from from_value
+    finally:
+        value = None
+"""
+    )
 else:
+
     def raise_from(value, from_value):
         raise value
 
 
 print_ = getattr(moves.builtins, "print", None)
 if print_ is None:
+
     def print_(*args, **kwargs):
         """The new-style print function for Python 2.4 and 2.5."""
         fp = kwargs.pop("file", sys.stdout)
@@ -730,14 +801,17 @@ if print_ is None:
             if not isinstance(data, basestring):
                 data = str(data)
             # If the file has an encoding, encode unicode with it.
-            if (isinstance(fp, file) and
-                    isinstance(data, unicode) and
-                    fp.encoding is not None):
+            if (
+                isinstance(fp, file)
+                and isinstance(data, unicode)
+                and fp.encoding is not None
+            ):
                 errors = getattr(fp, "errors", None)
                 if errors is None:
                     errors = "strict"
                 data = data.encode(fp.encoding, errors)
             fp.write(data)
+
         want_unicode = False
         sep = kwargs.pop("sep", None)
         if sep is not None:
@@ -773,6 +847,8 @@ if print_ is None:
                 write(sep)
             write(arg)
         write(end)
+
+
 if sys.version_info[:2] < (3, 3):
     _print = print_
 
@@ -783,16 +859,24 @@ if sys.version_info[:2] < (3, 3):
         if flush and fp is not None:
             fp.flush()
 
+
 _add_doc(reraise, """Reraise an exception.""")
 
 if sys.version_info[0:2] < (3, 4):
-    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
-              updated=functools.WRAPPER_UPDATES):
+
+    def wraps(
+        wrapped,
+        assigned=functools.WRAPPER_ASSIGNMENTS,
+        updated=functools.WRAPPER_UPDATES,
+    ):
         def wrapper(f):
             f = functools.wraps(wrapped, assigned, updated)(f)
             f.__wrapped__ = wrapped
             return f
+
         return wrapper
+
+
 else:
     wraps = functools.wraps
 
@@ -802,29 +886,95 @@ def with_metaclass(meta, *bases):
     # This requires a bit of explanation: the basic idea is to make a dummy
     # metaclass for one level of class instantiation that replaces itself with
     # the actual metaclass.
-    class metaclass(meta):
-
+    class metaclass(type):
         def __new__(cls, name, this_bases, d):
             return meta(name, bases, d)
-    return type.__new__(metaclass, 'temporary_class', (), {})
+
+        @classmethod
+        def __prepare__(cls, name, this_bases):
+            return meta.__prepare__(name, bases)
+
+    return type.__new__(metaclass, "temporary_class", (), {})
 
 
 def add_metaclass(metaclass):
     """Class decorator for creating a class with a metaclass."""
+
     def wrapper(cls):
         orig_vars = cls.__dict__.copy()
-        slots = orig_vars.get('__slots__')
+        slots = orig_vars.get("__slots__")
         if slots is not None:
             if isinstance(slots, str):
                 slots = [slots]
             for slots_var in slots:
                 orig_vars.pop(slots_var)
-        orig_vars.pop('__dict__', None)
-        orig_vars.pop('__weakref__', None)
+        orig_vars.pop("__dict__", None)
+        orig_vars.pop("__weakref__", None)
+        if hasattr(cls, "__qualname__"):
+            orig_vars["__qualname__"] = cls.__qualname__
         return metaclass(cls.__name__, cls.__bases__, orig_vars)
+
     return wrapper
 
 
+def ensure_binary(s, encoding="utf-8", errors="strict"):
+    """Coerce **s** to six.binary_type.
+
+    For Python 2:
+      - `unicode` -> encoded to `str`
+      - `str` -> `str`
+
+    For Python 3:
+      - `str` -> encoded to `bytes`
+      - `bytes` -> `bytes`
+    """
+    if isinstance(s, text_type):
+        return s.encode(encoding, errors)
+    elif isinstance(s, binary_type):
+        return s
+    else:
+        raise TypeError("not expecting type '%s'" % type(s))
+
+
+def ensure_str(s, encoding="utf-8", errors="strict"):
+    """Coerce *s* to `str`.
+
+    For Python 2:
+      - `unicode` -> encoded to `str`
+      - `str` -> `str`
+
+    For Python 3:
+      - `str` -> `str`
+      - `bytes` -> decoded to `str`
+    """
+    if not isinstance(s, (text_type, binary_type)):
+        raise TypeError("not expecting type '%s'" % type(s))
+    if PY2 and isinstance(s, text_type):
+        s = s.encode(encoding, errors)
+    elif PY3 and isinstance(s, binary_type):
+        s = s.decode(encoding, errors)
+    return s
+
+
+def ensure_text(s, encoding="utf-8", errors="strict"):
+    """Coerce *s* to six.text_type.
+
+    For Python 2:
+      - `unicode` -> `unicode`
+      - `str` -> `unicode`
+
+    For Python 3:
+      - `str` -> `str`
+      - `bytes` -> decoded to `str`
+    """
+    if isinstance(s, binary_type):
+        return s.decode(encoding, errors)
+    elif isinstance(s, text_type):
+        return s
+    else:
+        raise TypeError("not expecting type '%s'" % type(s))
+
+
 def python_2_unicode_compatible(klass):
     """
     A decorator that defines __unicode__ and __str__ methods under Python 2.
@@ -834,12 +984,13 @@ def python_2_unicode_compatible(klass):
     returning text and apply this decorator to the class.
     """
     if PY2:
-        if '__str__' not in klass.__dict__:
-            raise ValueError("@python_2_unicode_compatible cannot be applied "
-                             "to %s because it doesn't define __str__()." %
-                             klass.__name__)
+        if "__str__" not in klass.__dict__:
+            raise ValueError(
+                "@python_2_unicode_compatible cannot be applied "
+                "to %s because it doesn't define __str__()." % klass.__name__
+            )
         klass.__unicode__ = klass.__str__
-        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+        klass.__str__ = lambda self: self.__unicode__().encode("utf-8")
     return klass
 
 
@@ -859,8 +1010,10 @@ if sys.meta_path:
         # be floating around. Therefore, we can't use isinstance() to check for
         # the six meta path importer, since the other six instance will have
         # inserted an importer with different class.
-        if (type(importer).__name__ == "_SixMetaPathImporter" and
-                importer.name == __name__):
+        if (
+            type(importer).__name__ == "_SixMetaPathImporter"
+            and importer.name == __name__
+        ):
             del sys.meta_path[i]
             break
     del i, importer
diff --git a/lib/urllib3/packages/ssl_match_hostname/__init__.py b/lib/urllib3/packages/ssl_match_hostname/__init__.py
index d6594eb2..75b6bb1c 100644
--- a/lib/urllib3/packages/ssl_match_hostname/__init__.py
+++ b/lib/urllib3/packages/ssl_match_hostname/__init__.py
@@ -16,4 +16,4 @@ except ImportError:
         from ._implementation import CertificateError, match_hostname
 
 # Not needed, but documenting what we provide.
-__all__ = ('CertificateError', 'match_hostname')
+__all__ = ("CertificateError", "match_hostname")
diff --git a/lib/urllib3/packages/ssl_match_hostname/_implementation.py b/lib/urllib3/packages/ssl_match_hostname/_implementation.py
index 1fd42f38..2d8e7a17 100644
--- a/lib/urllib3/packages/ssl_match_hostname/_implementation.py
+++ b/lib/urllib3/packages/ssl_match_hostname/_implementation.py
@@ -9,14 +9,13 @@ import sys
 # ipaddress has been backported to 2.6+ in pypi.  If it is installed on the
 # system, use it to handle IPAddress ServerAltnames (this was added in
 # python-3.5) otherwise only do DNS matching.  This allows
-# backports.ssl_match_hostname to continue to be used all the way back to
-# python-2.4.
+# backports.ssl_match_hostname to continue to be used in Python 2.7.
 try:
     import ipaddress
 except ImportError:
     ipaddress = None
 
-__version__ = '3.5.0.1'
+__version__ = "3.5.0.1"
 
 
 class CertificateError(ValueError):
@@ -34,18 +33,19 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
 
     # Ported from python3-syntax:
     # leftmost, *remainder = dn.split(r'.')
-    parts = dn.split(r'.')
+    parts = dn.split(r".")
     leftmost = parts[0]
     remainder = parts[1:]
 
-    wildcards = leftmost.count('*')
+    wildcards = leftmost.count("*")
     if wildcards > max_wildcards:
         # Issue #17980: avoid denials of service by refusing more
         # than one wildcard per fragment.  A survey of established
         # policy among SSL implementations showed it to be a
         # reasonable choice.
         raise CertificateError(
-            "too many wildcards in certificate DNS name: " + repr(dn))
+            "too many wildcards in certificate DNS name: " + repr(dn)
+        )
 
     # speed up common case w/o wildcards
     if not wildcards:
@@ -54,11 +54,11 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
     # RFC 6125, section 6.4.3, subitem 1.
     # The client SHOULD NOT attempt to match a presented identifier in which
     # the wildcard character comprises a label other than the left-most label.
-    if leftmost == '*':
+    if leftmost == "*":
         # When '*' is a fragment by itself, it matches a non-empty dotless
         # fragment.
-        pats.append('[^.]+')
-    elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
+        pats.append("[^.]+")
+    elif leftmost.startswith("xn--") or hostname.startswith("xn--"):
         # RFC 6125, section 6.4.3, subitem 3.
         # The client SHOULD NOT attempt to match a presented identifier
         # where the wildcard character is embedded within an A-label or
@@ -66,21 +66,22 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
         pats.append(re.escape(leftmost))
     else:
         # Otherwise, '*' matches any dotless string, e.g. www*
-        pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
+        pats.append(re.escape(leftmost).replace(r"\*", "[^.]*"))
 
     # add the remaining fragments, ignore any wildcards
     for frag in remainder:
         pats.append(re.escape(frag))
 
-    pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+    pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE)
     return pat.match(hostname)
 
 
 def _to_unicode(obj):
     if isinstance(obj, str) and sys.version_info < (3,):
-        obj = unicode(obj, encoding='ascii', errors='strict')
+        obj = unicode(obj, encoding="ascii", errors="strict")
     return obj
 
+
 def _ipaddress_match(ipname, host_ip):
     """Exact matching of IP addresses.
 
@@ -102,9 +103,11 @@ def match_hostname(cert, hostname):
     returns nothing.
     """
     if not cert:
-        raise ValueError("empty or no certificate, match_hostname needs a "
-                         "SSL socket or SSL context with either "
-                         "CERT_OPTIONAL or CERT_REQUIRED")
+        raise ValueError(
+            "empty or no certificate, match_hostname needs a "
+            "SSL socket or SSL context with either "
+            "CERT_OPTIONAL or CERT_REQUIRED"
+        )
     try:
         # Divergence from upstream: ipaddress can't handle byte str
         host_ip = ipaddress.ip_address(_to_unicode(hostname))
@@ -123,35 +126,37 @@ def match_hostname(cert, hostname):
         else:
             raise
     dnsnames = []
-    san = cert.get('subjectAltName', ())
+    san = cert.get("subjectAltName", ())
     for key, value in san:
-        if key == 'DNS':
+        if key == "DNS":
             if host_ip is None and _dnsname_match(value, hostname):
                 return
             dnsnames.append(value)
-        elif key == 'IP Address':
+        elif key == "IP Address":
             if host_ip is not None and _ipaddress_match(value, host_ip):
                 return
             dnsnames.append(value)
     if not dnsnames:
         # The subject is only checked when there is no dNSName entry
         # in subjectAltName
-        for sub in cert.get('subject', ()):
+        for sub in cert.get("subject", ()):
             for key, value in sub:
                 # XXX according to RFC 2818, the most specific Common Name
                 # must be used.
-                if key == 'commonName':
+                if key == "commonName":
                     if _dnsname_match(value, hostname):
                         return
                     dnsnames.append(value)
     if len(dnsnames) > 1:
-        raise CertificateError("hostname %r "
-            "doesn't match either of %s"
-            % (hostname, ', '.join(map(repr, dnsnames))))
+        raise CertificateError(
+            "hostname %r "
+            "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames)))
+        )
     elif len(dnsnames) == 1:
-        raise CertificateError("hostname %r "
-            "doesn't match %r"
-            % (hostname, dnsnames[0]))
+        raise CertificateError(
+            "hostname %r " "doesn't match %r" % (hostname, dnsnames[0])
+        )
     else:
-        raise CertificateError("no appropriate commonName or "
-            "subjectAltName fields were found")
+        raise CertificateError(
+            "no appropriate commonName or " "subjectAltName fields were found"
+        )
diff --git a/lib/urllib3/poolmanager.py b/lib/urllib3/poolmanager.py
index 4ae91744..242a2f82 100644
--- a/lib/urllib3/poolmanager.py
+++ b/lib/urllib3/poolmanager.py
@@ -7,51 +7,62 @@ from ._collections import RecentlyUsedContainer
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
 from .connectionpool import port_by_scheme
 from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown
+from .packages import six
 from .packages.six.moves.urllib.parse import urljoin
 from .request import RequestMethods
 from .util.url import parse_url
 from .util.retry import Retry
 
 
-__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
+__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
 
 
 log = logging.getLogger(__name__)
 
-SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
-                'ssl_version', 'ca_cert_dir', 'ssl_context')
+SSL_KEYWORDS = (
+    "key_file",
+    "cert_file",
+    "cert_reqs",
+    "ca_certs",
+    "ssl_version",
+    "ca_cert_dir",
+    "ssl_context",
+    "key_password",
+)
 
 # All known keyword arguments that could be provided to the pool manager, its
 # pools, or the underlying connections. This is used to construct a pool key.
 _key_fields = (
-    'key_scheme',  # str
-    'key_host',  # str
-    'key_port',  # int
-    'key_timeout',  # int or float or Timeout
-    'key_retries',  # int or Retry
-    'key_strict',  # bool
-    'key_block',  # bool
-    'key_source_address',  # str
-    'key_key_file',  # str
-    'key_cert_file',  # str
-    'key_cert_reqs',  # str
-    'key_ca_certs',  # str
-    'key_ssl_version',  # str
-    'key_ca_cert_dir',  # str
-    'key_ssl_context',  # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
-    'key_maxsize',  # int
-    'key_headers',  # dict
-    'key__proxy',  # parsed proxy url
-    'key__proxy_headers',  # dict
-    'key_socket_options',  # list of (level (int), optname (int), value (int or str)) tuples
-    'key__socks_options',  # dict
-    'key_assert_hostname',  # bool or string
-    'key_assert_fingerprint',  # str
+    "key_scheme",  # str
+    "key_host",  # str
+    "key_port",  # int
+    "key_timeout",  # int or float or Timeout
+    "key_retries",  # int or Retry
+    "key_strict",  # bool
+    "key_block",  # bool
+    "key_source_address",  # str
+    "key_key_file",  # str
+    "key_key_password",  # str
+    "key_cert_file",  # str
+    "key_cert_reqs",  # str
+    "key_ca_certs",  # str
+    "key_ssl_version",  # str
+    "key_ca_cert_dir",  # str
+    "key_ssl_context",  # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
+    "key_maxsize",  # int
+    "key_headers",  # dict
+    "key__proxy",  # parsed proxy url
+    "key__proxy_headers",  # dict
+    "key_socket_options",  # list of (level (int), optname (int), value (int or str)) tuples
+    "key__socks_options",  # dict
+    "key_assert_hostname",  # bool or string
+    "key_assert_fingerprint",  # str
+    "key_server_hostname",  # str
 )
 
 #: The namedtuple class used to construct keys for the connection pool.
 #: All custom key schemes should include the fields in this key at a minimum.
-PoolKey = collections.namedtuple('PoolKey', _key_fields)
+PoolKey = collections.namedtuple("PoolKey", _key_fields)
 
 
 def _default_key_normalizer(key_class, request_context):
@@ -76,24 +87,24 @@ def _default_key_normalizer(key_class, request_context):
     """
     # Since we mutate the dictionary, make a copy first
     context = request_context.copy()
-    context['scheme'] = context['scheme'].lower()
-    context['host'] = context['host'].lower()
+    context["scheme"] = context["scheme"].lower()
+    context["host"] = context["host"].lower()
 
     # These are both dictionaries and need to be transformed into frozensets
-    for key in ('headers', '_proxy_headers', '_socks_options'):
+    for key in ("headers", "_proxy_headers", "_socks_options"):
         if key in context and context[key] is not None:
             context[key] = frozenset(context[key].items())
 
     # The socket_options key may be a list and needs to be transformed into a
     # tuple.
-    socket_opts = context.get('socket_options')
+    socket_opts = context.get("socket_options")
     if socket_opts is not None:
-        context['socket_options'] = tuple(socket_opts)
+        context["socket_options"] = tuple(socket_opts)
 
     # Map the kwargs to the names in the namedtuple - this is necessary since
     # namedtuples can't have fields starting with '_'.
     for key in list(context.keys()):
-        context['key_' + key] = context.pop(key)
+        context["key_" + key] = context.pop(key)
 
     # Default to ``None`` for keys missing from the context
     for field in key_class._fields:
@@ -108,14 +119,11 @@ def _default_key_normalizer(key_class, request_context):
 #: Each PoolManager makes a copy of this dictionary so they can be configured
 #: globally here, or individually on the instance.
 key_fn_by_scheme = {
-    'http': functools.partial(_default_key_normalizer, PoolKey),
-    'https': functools.partial(_default_key_normalizer, PoolKey),
+    "http": functools.partial(_default_key_normalizer, PoolKey),
+    "https": functools.partial(_default_key_normalizer, PoolKey),
 }
 
-pool_classes_by_scheme = {
-    'http': HTTPConnectionPool,
-    'https': HTTPSConnectionPool,
-}
+pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
 
 
 class PoolManager(RequestMethods):
@@ -151,8 +159,7 @@ class PoolManager(RequestMethods):
     def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
         RequestMethods.__init__(self, headers)
         self.connection_pool_kw = connection_pool_kw
-        self.pools = RecentlyUsedContainer(num_pools,
-                                           dispose_func=lambda p: p.close())
+        self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close())
 
         # Locally set the pool classes and keys so other PoolManagers can
         # override them.
@@ -185,10 +192,10 @@ class PoolManager(RequestMethods):
         # this function has historically only used the scheme, host, and port
         # in the positional args. When an API change is acceptable these can
         # be removed.
-        for key in ('scheme', 'host', 'port'):
+        for key in ("scheme", "host", "port"):
             request_context.pop(key, None)
 
-        if scheme == 'http':
+        if scheme == "http":
             for kw in SSL_KEYWORDS:
                 request_context.pop(kw, None)
 
@@ -203,7 +210,7 @@ class PoolManager(RequestMethods):
         """
         self.pools.clear()
 
-    def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None):
+    def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
         """
         Get a :class:`ConnectionPool` based on the host, port, and scheme.
 
@@ -218,11 +225,11 @@ class PoolManager(RequestMethods):
             raise LocationValueError("No host specified.")
 
         request_context = self._merge_pool_kwargs(pool_kwargs)
-        request_context['scheme'] = scheme or 'http'
+        request_context["scheme"] = scheme or "http"
         if not port:
-            port = port_by_scheme.get(request_context['scheme'].lower(), 80)
-        request_context['port'] = port
-        request_context['host'] = host
+            port = port_by_scheme.get(request_context["scheme"].lower(), 80)
+        request_context["port"] = port
+        request_context["host"] = host
 
         return self.connection_from_context(request_context)
 
@@ -233,7 +240,7 @@ class PoolManager(RequestMethods):
         ``request_context`` must at least contain the ``scheme`` key and its
         value must be a key in ``key_fn_by_scheme`` instance variable.
         """
-        scheme = request_context['scheme'].lower()
+        scheme = request_context["scheme"].lower()
         pool_key_constructor = self.key_fn_by_scheme[scheme]
         pool_key = pool_key_constructor(request_context)
 
@@ -255,9 +262,9 @@ class PoolManager(RequestMethods):
                 return pool
 
             # Make a fresh ConnectionPool of the desired type
-            scheme = request_context['scheme']
-            host = request_context['host']
-            port = request_context['port']
+            scheme = request_context["scheme"]
+            host = request_context["host"]
+            port = request_context["port"]
             pool = self._new_pool(scheme, host, port, request_context=request_context)
             self.pools[pool_key] = pool
 
@@ -275,8 +282,9 @@ class PoolManager(RequestMethods):
         not used.
         """
         u = parse_url(url)
-        return self.connection_from_host(u.host, port=u.port, scheme=u.scheme,
-                                         pool_kwargs=pool_kwargs)
+        return self.connection_from_host(
+            u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
+        )
 
     def _merge_pool_kwargs(self, override):
         """
@@ -310,10 +318,11 @@ class PoolManager(RequestMethods):
         u = parse_url(url)
         conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
 
-        kw['assert_same_host'] = False
-        kw['redirect'] = False
-        if 'headers' not in kw:
-            kw['headers'] = self.headers
+        kw["assert_same_host"] = False
+        kw["redirect"] = False
+
+        if "headers" not in kw:
+            kw["headers"] = self.headers.copy()
 
         if self.proxy is not None and u.scheme == "http":
             response = conn.urlopen(method, url, **kw)
@@ -329,12 +338,23 @@ class PoolManager(RequestMethods):
 
         # RFC 7231, Section 6.4.4
         if response.status == 303:
-            method = 'GET'
+            method = "GET"
 
-        retries = kw.get('retries')
+        retries = kw.get("retries")
         if not isinstance(retries, Retry):
             retries = Retry.from_int(retries, redirect=redirect)
 
+        # Strip headers marked as unsafe to forward to the redirected location.
+        # Check remove_headers_on_redirect to avoid a potential network call within
+        # conn.is_same_host() which may use socket.gethostbyname() in the future.
+        if retries.remove_headers_on_redirect and not conn.is_same_host(
+            redirect_location
+        ):
+            headers = list(six.iterkeys(kw["headers"]))
+            for header in headers:
+                if header.lower() in retries.remove_headers_on_redirect:
+                    kw["headers"].pop(header, None)
+
         try:
             retries = retries.increment(method, url, response=response, _pool=conn)
         except MaxRetryError:
@@ -342,8 +362,8 @@ class PoolManager(RequestMethods):
                 raise
             return response
 
-        kw['retries'] = retries
-        kw['redirect'] = redirect
+        kw["retries"] = retries
+        kw["redirect"] = redirect
 
         log.info("Redirecting %s -> %s", url, redirect_location)
         return self.urlopen(method, redirect_location, **kw)
@@ -358,7 +378,7 @@ class ProxyManager(PoolManager):
         The URL of the proxy to be used.
 
     :param proxy_headers:
-        A dictionary contaning headers that will be sent to the proxy. In case
+        A dictionary containing headers that will be sent to the proxy. In case
         of HTTP they are being sent with each request, while in the
         HTTPS/CONNECT case they are sent only once. Could be used for proxy
         authentication.
@@ -376,12 +396,21 @@ class ProxyManager(PoolManager):
 
     """
 
-    def __init__(self, proxy_url, num_pools=10, headers=None,
-                 proxy_headers=None, **connection_pool_kw):
+    def __init__(
+        self,
+        proxy_url,
+        num_pools=10,
+        headers=None,
+        proxy_headers=None,
+        **connection_pool_kw
+    ):
 
         if isinstance(proxy_url, HTTPConnectionPool):
-            proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host,
-                                        proxy_url.port)
+            proxy_url = "%s://%s:%i" % (
+                proxy_url.scheme,
+                proxy_url.host,
+                proxy_url.port,
+            )
         proxy = parse_url(proxy_url)
         if not proxy.port:
             port = port_by_scheme.get(proxy.scheme, 80)
@@ -393,30 +422,31 @@ class ProxyManager(PoolManager):
         self.proxy = proxy
         self.proxy_headers = proxy_headers or {}
 
-        connection_pool_kw['_proxy'] = self.proxy
-        connection_pool_kw['_proxy_headers'] = self.proxy_headers
+        connection_pool_kw["_proxy"] = self.proxy
+        connection_pool_kw["_proxy_headers"] = self.proxy_headers
 
-        super(ProxyManager, self).__init__(
-            num_pools, headers, **connection_pool_kw)
+        super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
 
-    def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None):
+    def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
         if scheme == "https":
             return super(ProxyManager, self).connection_from_host(
-                host, port, scheme, pool_kwargs=pool_kwargs)
+                host, port, scheme, pool_kwargs=pool_kwargs
+            )
 
         return super(ProxyManager, self).connection_from_host(
-            self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs)
+            self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
+        )
 
     def _set_proxy_headers(self, url, headers=None):
         """
         Sets headers needed by proxies: specifically, the Accept and Host
         headers. Only sets headers not provided by the user.
         """
-        headers_ = {'Accept': '*/*'}
+        headers_ = {"Accept": "*/*"}
 
         netloc = parse_url(url).netloc
         if netloc:
-            headers_['Host'] = netloc
+            headers_["Host"] = netloc
 
         if headers:
             headers_.update(headers)
@@ -430,8 +460,8 @@ class ProxyManager(PoolManager):
             # For proxied HTTPS requests, httplib sets the necessary headers
             # on the CONNECT to the proxy. For HTTP, we'll definitely
             # need to set 'Host' at the very least.
-            headers = kw.get('headers', self.headers)
-            kw['headers'] = self._set_proxy_headers(url, headers)
+            headers = kw.get("headers", self.headers)
+            kw["headers"] = self._set_proxy_headers(url, headers)
 
         return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
 
diff --git a/lib/urllib3/request.py b/lib/urllib3/request.py
index c0fddff0..55f160bb 100644
--- a/lib/urllib3/request.py
+++ b/lib/urllib3/request.py
@@ -4,7 +4,7 @@ from .filepost import encode_multipart_formdata
 from .packages.six.moves.urllib.parse import urlencode
 
 
-__all__ = ['RequestMethods']
+__all__ = ["RequestMethods"]
 
 
 class RequestMethods(object):
@@ -36,16 +36,25 @@ class RequestMethods(object):
         explicitly.
     """
 
-    _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS'])
+    _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
 
     def __init__(self, headers=None):
         self.headers = headers or {}
 
-    def urlopen(self, method, url, body=None, headers=None,
-                encode_multipart=True, multipart_boundary=None,
-                **kw):  # Abstract
-        raise NotImplemented("Classes extending RequestMethods must implement "
-                             "their own ``urlopen`` method.")
+    def urlopen(
+        self,
+        method,
+        url,
+        body=None,
+        headers=None,
+        encode_multipart=True,
+        multipart_boundary=None,
+        **kw
+    ):  # Abstract
+        raise NotImplementedError(
+            "Classes extending RequestMethods must implement "
+            "their own ``urlopen`` method."
+        )
 
     def request(self, method, url, fields=None, headers=None, **urlopen_kw):
         """
@@ -60,17 +69,18 @@ class RequestMethods(object):
         """
         method = method.upper()
 
-        if method in self._encode_url_methods:
-            return self.request_encode_url(method, url, fields=fields,
-                                           headers=headers,
-                                           **urlopen_kw)
-        else:
-            return self.request_encode_body(method, url, fields=fields,
-                                            headers=headers,
-                                            **urlopen_kw)
+        urlopen_kw["request_url"] = url
 
-    def request_encode_url(self, method, url, fields=None, headers=None,
-                           **urlopen_kw):
+        if method in self._encode_url_methods:
+            return self.request_encode_url(
+                method, url, fields=fields, headers=headers, **urlopen_kw
+            )
+        else:
+            return self.request_encode_body(
+                method, url, fields=fields, headers=headers, **urlopen_kw
+            )
+
+    def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw):
         """
         Make a request using :meth:`urlopen` with the ``fields`` encoded in
         the url. This is useful for request methods like GET, HEAD, DELETE, etc.
@@ -78,17 +88,24 @@ class RequestMethods(object):
         if headers is None:
             headers = self.headers
 
-        extra_kw = {'headers': headers}
+        extra_kw = {"headers": headers}
         extra_kw.update(urlopen_kw)
 
         if fields:
-            url += '?' + urlencode(fields)
+            url += "?" + urlencode(fields)
 
         return self.urlopen(method, url, **extra_kw)
 
-    def request_encode_body(self, method, url, fields=None, headers=None,
-                            encode_multipart=True, multipart_boundary=None,
-                            **urlopen_kw):
+    def request_encode_body(
+        self,
+        method,
+        url,
+        fields=None,
+        headers=None,
+        encode_multipart=True,
+        multipart_boundary=None,
+        **urlopen_kw
+    ):
         """
         Make a request using :meth:`urlopen` with the ``fields`` encoded in
         the body. This is useful for request methods like POST, PUT, PATCH, etc.
@@ -117,7 +134,7 @@ class RequestMethods(object):
             }
 
         When uploading a file, providing a filename (the first parameter of the
-        tuple) is optional but recommended to best mimick behavior of browsers.
+        tuple) is optional but recommended to best mimic behavior of browsers.
 
         Note that if ``headers`` are supplied, the 'Content-Type' header will
         be overwritten because it depends on the dynamic random boundary string
@@ -127,22 +144,28 @@ class RequestMethods(object):
         if headers is None:
             headers = self.headers
 
-        extra_kw = {'headers': {}}
+        extra_kw = {"headers": {}}
 
         if fields:
-            if 'body' in urlopen_kw:
+            if "body" in urlopen_kw:
                 raise TypeError(
-                    "request got values for both 'fields' and 'body', can only specify one.")
+                    "request got values for both 'fields' and 'body', can only specify one."
+                )
 
             if encode_multipart:
-                body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary)
+                body, content_type = encode_multipart_formdata(
+                    fields, boundary=multipart_boundary
+                )
             else:
-                body, content_type = urlencode(fields), 'application/x-www-form-urlencoded'
+                body, content_type = (
+                    urlencode(fields),
+                    "application/x-www-form-urlencoded",
+                )
 
-            extra_kw['body'] = body
-            extra_kw['headers'] = {'Content-Type': content_type}
+            extra_kw["body"] = body
+            extra_kw["headers"] = {"Content-Type": content_type}
 
-        extra_kw['headers'].update(headers)
+        extra_kw["headers"].update(headers)
         extra_kw.update(urlopen_kw)
 
         return self.urlopen(method, url, **extra_kw)
diff --git a/lib/urllib3/response.py b/lib/urllib3/response.py
index d3e5a1e6..adc321e7 100644
--- a/lib/urllib3/response.py
+++ b/lib/urllib3/response.py
@@ -6,12 +6,22 @@ import logging
 from socket import timeout as SocketTimeout
 from socket import error as SocketError
 
+try:
+    import brotli
+except ImportError:
+    brotli = None
+
 from ._collections import HTTPHeaderDict
 from .exceptions import (
-    BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
-    ResponseNotChunked, IncompleteRead, InvalidHeader
+    BodyNotHttplibCompatible,
+    ProtocolError,
+    DecodeError,
+    ReadTimeoutError,
+    ResponseNotChunked,
+    IncompleteRead,
+    InvalidHeader,
 )
-from .packages.six import string_types as basestring, binary_type, PY3
+from .packages.six import string_types as basestring, PY3
 from .packages.six.moves import http_client as httplib
 from .connection import HTTPException, BaseSSLError
 from .util.response import is_fp_closed, is_response_to_head
@@ -20,10 +30,9 @@ log = logging.getLogger(__name__)
 
 
 class DeflateDecoder(object):
-
     def __init__(self):
         self._first_try = True
-        self._data = binary_type()
+        self._data = b""
         self._obj = zlib.decompressobj()
 
     def __getattr__(self, name):
@@ -52,24 +61,94 @@ class DeflateDecoder(object):
                 self._data = None
 
 
-class GzipDecoder(object):
+class GzipDecoderState(object):
 
+    FIRST_MEMBER = 0
+    OTHER_MEMBERS = 1
+    SWALLOW_DATA = 2
+
+
+class GzipDecoder(object):
     def __init__(self):
         self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+        self._state = GzipDecoderState.FIRST_MEMBER
 
     def __getattr__(self, name):
         return getattr(self._obj, name)
 
     def decompress(self, data):
-        if not data:
-            return data
-        return self._obj.decompress(data)
+        ret = bytearray()
+        if self._state == GzipDecoderState.SWALLOW_DATA or not data:
+            return bytes(ret)
+        while True:
+            try:
+                ret += self._obj.decompress(data)
+            except zlib.error:
+                previous_state = self._state
+                # Ignore data after the first error
+                self._state = GzipDecoderState.SWALLOW_DATA
+                if previous_state == GzipDecoderState.OTHER_MEMBERS:
+                    # Allow trailing garbage acceptable in other gzip clients
+                    return bytes(ret)
+                raise
+            data = self._obj.unused_data
+            if not data:
+                return bytes(ret)
+            self._state = GzipDecoderState.OTHER_MEMBERS
+            self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+
+if brotli is not None:
+
+    class BrotliDecoder(object):
+        # Supports both 'brotlipy' and 'Brotli' packages
+        # since they share an import name. The top branches
+        # are for 'brotlipy' and bottom branches for 'Brotli'
+        def __init__(self):
+            self._obj = brotli.Decompressor()
+
+        def decompress(self, data):
+            if hasattr(self._obj, "decompress"):
+                return self._obj.decompress(data)
+            return self._obj.process(data)
+
+        def flush(self):
+            if hasattr(self._obj, "flush"):
+                return self._obj.flush()
+            return b""
+
+
+class MultiDecoder(object):
+    """
+    From RFC7231:
+        If one or more encodings have been applied to a representation, the
+        sender that applied the encodings MUST generate a Content-Encoding
+        header field that lists the content codings in the order in which
+        they were applied.
+    """
+
+    def __init__(self, modes):
+        self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
+
+    def flush(self):
+        return self._decoders[0].flush()
+
+    def decompress(self, data):
+        for d in reversed(self._decoders):
+            data = d.decompress(data)
+        return data
 
 
 def _get_decoder(mode):
-    if mode == 'gzip':
+    if "," in mode:
+        return MultiDecoder(mode)
+
+    if mode == "gzip":
         return GzipDecoder()
 
+    if brotli is not None and mode == "br":
+        return BrotliDecoder()
+
     return DeflateDecoder()
 
 
@@ -89,9 +168,8 @@ class HTTPResponse(io.IOBase):
         If True, the response's body will be preloaded during construction.
 
     :param decode_content:
-        If True, attempts to decode specific content-encoding's based on headers
-        (like 'gzip' and 'deflate') will be skipped and raw data will be used
-        instead.
+        If True, will attempt to decode the body based on the
+        'content-encoding' header.
 
     :param original_response:
         When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
@@ -107,13 +185,31 @@ class HTTPResponse(io.IOBase):
         value of Content-Length header, if present. Otherwise, raise error.
     """
 
-    CONTENT_DECODERS = ['gzip', 'deflate']
+    CONTENT_DECODERS = ["gzip", "deflate"]
+    if brotli is not None:
+        CONTENT_DECODERS += ["br"]
     REDIRECT_STATUSES = [301, 302, 303, 307, 308]
 
-    def __init__(self, body='', headers=None, status=0, version=0, reason=None,
-                 strict=0, preload_content=True, decode_content=True,
-                 original_response=None, pool=None, connection=None,
-                 retries=None, enforce_content_length=False, request_method=None):
+    def __init__(
+        self,
+        body="",
+        headers=None,
+        status=0,
+        version=0,
+        reason=None,
+        strict=0,
+        preload_content=True,
+        decode_content=True,
+        original_response=None,
+        pool=None,
+        connection=None,
+        msg=None,
+        retries=None,
+        enforce_content_length=False,
+        request_method=None,
+        request_url=None,
+        auto_close=True,
+    ):
 
         if isinstance(headers, HTTPHeaderDict):
             self.headers = headers
@@ -126,26 +222,29 @@ class HTTPResponse(io.IOBase):
         self.decode_content = decode_content
         self.retries = retries
         self.enforce_content_length = enforce_content_length
+        self.auto_close = auto_close
 
         self._decoder = None
         self._body = None
         self._fp = None
         self._original_response = original_response
         self._fp_bytes_read = 0
+        self.msg = msg
+        self._request_url = request_url
 
-        if body and isinstance(body, (basestring, binary_type)):
+        if body and isinstance(body, (basestring, bytes)):
             self._body = body
 
         self._pool = pool
         self._connection = connection
 
-        if hasattr(body, 'read'):
+        if hasattr(body, "read"):
             self._fp = body
 
         # Are we using the chunked-style of transfer encoding?
         self.chunked = False
         self.chunk_left = None
-        tr_enc = self.headers.get('transfer-encoding', '').lower()
+        tr_enc = self.headers.get("transfer-encoding", "").lower()
         # Don't incur the penalty of creating a list and then discarding it
         encodings = (enc.strip() for enc in tr_enc.split(","))
         if "chunked" in encodings:
@@ -167,7 +266,7 @@ class HTTPResponse(io.IOBase):
             location. ``False`` if not a redirect status code.
         """
         if self.status in self.REDIRECT_STATUSES:
-            return self.headers.get('location')
+            return self.headers.get("location")
 
         return False
 
@@ -191,6 +290,9 @@ class HTTPResponse(io.IOBase):
     def connection(self):
         return self._connection
 
+    def isclosed(self):
+        return is_fp_closed(self._fp)
+
     def tell(self):
         """
         Obtain the number of bytes pulled over the wire so far. May differ from
@@ -203,30 +305,34 @@ class HTTPResponse(io.IOBase):
         """
         Set initial length value for Response content if available.
         """
-        length = self.headers.get('content-length')
+        length = self.headers.get("content-length")
 
-        if length is not None and self.chunked:
-            # This Response will fail with an IncompleteRead if it can't be
-            # received as chunked. This method falls back to attempt reading
-            # the response before raising an exception.
-            log.warning("Received response with both Content-Length and "
-                        "Transfer-Encoding set. This is expressly forbidden "
-                        "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
-                        "attempting to process response as Transfer-Encoding: "
-                        "chunked.")
-            return None
+        if length is not None:
+            if self.chunked:
+                # This Response will fail with an IncompleteRead if it can't be
+                # received as chunked. This method falls back to attempt reading
+                # the response before raising an exception.
+                log.warning(
+                    "Received response with both Content-Length and "
+                    "Transfer-Encoding set. This is expressly forbidden "
+                    "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
+                    "attempting to process response as Transfer-Encoding: "
+                    "chunked."
+                )
+                return None
 
-        elif length is not None:
             try:
                 # RFC 7230 section 3.3.2 specifies multiple content lengths can
                 # be sent in a single Content-Length header
                 # (e.g. Content-Length: 42, 42). This line ensures the values
                 # are all valid ints and that as long as the `set` length is 1,
                 # all values are the same. Otherwise, the header is invalid.
-                lengths = set([int(val) for val in length.split(',')])
+                lengths = set([int(val) for val in length.split(",")])
                 if len(lengths) > 1:
-                    raise InvalidHeader("Content-Length contained multiple "
-                                        "unmatching values (%s)" % length)
+                    raise InvalidHeader(
+                        "Content-Length contained multiple "
+                        "unmatching values (%s)" % length
+                    )
                 length = lengths.pop()
             except ValueError:
                 length = None
@@ -242,7 +348,7 @@ class HTTPResponse(io.IOBase):
             status = 0
 
         # Check for responses that shouldn't include a body
-        if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD':
+        if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
             length = 0
 
         return length
@@ -253,24 +359,41 @@ class HTTPResponse(io.IOBase):
         """
         # Note: content-encoding value should be case-insensitive, per RFC 7230
         # Section 3.2
-        content_encoding = self.headers.get('content-encoding', '').lower()
-        if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
-            self._decoder = _get_decoder(content_encoding)
+        content_encoding = self.headers.get("content-encoding", "").lower()
+        if self._decoder is None:
+            if content_encoding in self.CONTENT_DECODERS:
+                self._decoder = _get_decoder(content_encoding)
+            elif "," in content_encoding:
+                encodings = [
+                    e.strip()
+                    for e in content_encoding.split(",")
+                    if e.strip() in self.CONTENT_DECODERS
+                ]
+                if len(encodings):
+                    self._decoder = _get_decoder(content_encoding)
+
+    DECODER_ERROR_CLASSES = (IOError, zlib.error)
+    if brotli is not None:
+        DECODER_ERROR_CLASSES += (brotli.error,)
 
     def _decode(self, data, decode_content, flush_decoder):
         """
         Decode the data passed in and potentially flush the decoder.
         """
+        if not decode_content:
+            return data
+
         try:
-            if decode_content and self._decoder:
+            if self._decoder:
                 data = self._decoder.decompress(data)
-        except (IOError, zlib.error) as e:
-            content_encoding = self.headers.get('content-encoding', '').lower()
+        except self.DECODER_ERROR_CLASSES as e:
+            content_encoding = self.headers.get("content-encoding", "").lower()
             raise DecodeError(
                 "Received response with content-encoding: %s, but "
-                "failed to decode it." % content_encoding, e)
-
-        if flush_decoder and decode_content:
+                "failed to decode it." % content_encoding,
+                e,
+            )
+        if flush_decoder:
             data += self._flush_decoder()
 
         return data
@@ -281,10 +404,10 @@ class HTTPResponse(io.IOBase):
         being used.
         """
         if self._decoder:
-            buf = self._decoder.decompress(b'')
+            buf = self._decoder.decompress(b"")
             return buf + self._decoder.flush()
 
-        return b''
+        return b""
 
     @contextmanager
     def _error_catcher(self):
@@ -304,20 +427,20 @@ class HTTPResponse(io.IOBase):
             except SocketTimeout:
                 # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
                 # there is yet no clean way to get at it from this context.
-                raise ReadTimeoutError(self._pool, None, 'Read timed out.')
+                raise ReadTimeoutError(self._pool, None, "Read timed out.")
 
             except BaseSSLError as e:
                 # FIXME: Is there a better way to differentiate between SSLErrors?
-                if 'read operation timed out' not in str(e):  # Defensive:
+                if "read operation timed out" not in str(e):  # Defensive:
                     # This shouldn't happen but just in case we're missing an edge
                     # case, let's avoid swallowing SSL errors.
                     raise
 
-                raise ReadTimeoutError(self._pool, None, 'Read timed out.')
+                raise ReadTimeoutError(self._pool, None, "Read timed out.")
 
             except (HTTPException, SocketError) as e:
                 # This includes IncompleteRead.
-                raise ProtocolError('Connection broken: %r' % e, e)
+                raise ProtocolError("Connection broken: %r" % e, e)
 
             # If no exception is thrown, we should avoid cleaning up
             # unnecessarily.
@@ -372,17 +495,19 @@ class HTTPResponse(io.IOBase):
             return
 
         flush_decoder = False
-        data = None
+        fp_closed = getattr(self._fp, "closed", False)
 
         with self._error_catcher():
             if amt is None:
                 # cStringIO doesn't like amt=None
-                data = self._fp.read()
+                data = self._fp.read() if not fp_closed else b""
                 flush_decoder = True
             else:
                 cache_content = False
-                data = self._fp.read(amt)
-                if amt != 0 and not data:  # Platform-specific: Buggy versions of Python.
+                data = self._fp.read(amt) if not fp_closed else b""
+                if (
+                    amt != 0 and not data
+                ):  # Platform-specific: Buggy versions of Python.
                     # Close the connection when no data is returned
                     #
                     # This is redundant to what httplib/http.client _should_
@@ -392,7 +517,10 @@ class HTTPResponse(io.IOBase):
                     # no harm in redundantly calling close.
                     self._fp.close()
                     flush_decoder = True
-                    if self.enforce_content_length and self.length_remaining not in (0, None):
+                    if self.enforce_content_length and self.length_remaining not in (
+                        0,
+                        None,
+                    ):
                         # This is an edge case that httplib failed to cover due
                         # to concerns of backward compatibility. We're
                         # addressing it here to make sure IncompleteRead is
@@ -412,7 +540,7 @@ class HTTPResponse(io.IOBase):
 
         return data
 
-    def stream(self, amt=2**16, decode_content=None):
+    def stream(self, amt=2 ** 16, decode_content=None):
         """
         A generator wrapper for the read() method. A call will block until
         ``amt`` bytes have been read from the connection or until the
@@ -450,21 +578,24 @@ class HTTPResponse(io.IOBase):
         headers = r.msg
 
         if not isinstance(headers, HTTPHeaderDict):
-            if PY3:  # Python 3
+            if PY3:
                 headers = HTTPHeaderDict(headers.items())
-            else:  # Python 2
+            else:
+                # Python 2.7
                 headers = HTTPHeaderDict.from_httplib(headers)
 
         # HTTPResponse objects in Python 3 don't have a .strict attribute
-        strict = getattr(r, 'strict', 0)
-        resp = ResponseCls(body=r,
-                           headers=headers,
-                           status=r.status,
-                           version=r.version,
-                           reason=r.reason,
-                           strict=strict,
-                           original_response=r,
-                           **response_kw)
+        strict = getattr(r, "strict", 0)
+        resp = ResponseCls(
+            body=r,
+            headers=headers,
+            status=r.status,
+            version=r.version,
+            reason=r.reason,
+            strict=strict,
+            original_response=r,
+            **response_kw
+        )
         return resp
 
     # Backwards-compatibility methods for httplib.HTTPResponse
@@ -486,13 +617,18 @@ class HTTPResponse(io.IOBase):
         if self._connection:
             self._connection.close()
 
+        if not self.auto_close:
+            io.IOBase.close(self)
+
     @property
     def closed(self):
-        if self._fp is None:
+        if not self.auto_close:
+            return io.IOBase.closed.__get__(self)
+        elif self._fp is None:
             return True
-        elif hasattr(self._fp, 'isclosed'):
+        elif hasattr(self._fp, "isclosed"):
             return self._fp.isclosed()
-        elif hasattr(self._fp, 'closed'):
+        elif hasattr(self._fp, "closed"):
             return self._fp.closed
         else:
             return True
@@ -503,11 +639,17 @@ class HTTPResponse(io.IOBase):
         elif hasattr(self._fp, "fileno"):
             return self._fp.fileno()
         else:
-            raise IOError("The file-like object this HTTPResponse is wrapped "
-                          "around has no file descriptor")
+            raise IOError(
+                "The file-like object this HTTPResponse is wrapped "
+                "around has no file descriptor"
+            )
 
     def flush(self):
-        if self._fp is not None and hasattr(self._fp, 'flush'):
+        if (
+            self._fp is not None
+            and hasattr(self._fp, "flush")
+            and not getattr(self._fp, "closed", False)
+        ):
             return self._fp.flush()
 
     def readable(self):
@@ -520,7 +662,7 @@ class HTTPResponse(io.IOBase):
         if len(temp) == 0:
             return 0
         else:
-            b[:len(temp)] = temp
+            b[: len(temp)] = temp
             return len(temp)
 
     def supports_chunked_reads(self):
@@ -530,7 +672,7 @@ class HTTPResponse(io.IOBase):
         attribute. If it is present we assume it returns raw chunks as
         processed by read_chunked().
         """
-        return hasattr(self._fp, 'fp')
+        return hasattr(self._fp, "fp")
 
     def _update_chunk_length(self):
         # First, we'll figure out length of a chunk and then
@@ -538,7 +680,7 @@ class HTTPResponse(io.IOBase):
         if self.chunk_left is not None:
             return
         line = self._fp.fp.readline()
-        line = line.split(b';', 1)[0]
+        line = line.split(b";", 1)[0]
         try:
             self.chunk_left = int(line, 16)
         except ValueError:
@@ -573,6 +715,11 @@ class HTTPResponse(io.IOBase):
         Similar to :meth:`HTTPResponse.read`, but with an additional
         parameter: ``decode_content``.
 
+        :param amt:
+            How much of the content to read. If specified, caching is skipped
+            because it doesn't make sense to cache partial content as the full
+            response.
+
         :param decode_content:
             If True, will attempt to decode the body based on the
             'content-encoding' header.
@@ -582,25 +729,33 @@ class HTTPResponse(io.IOBase):
         if not self.chunked:
             raise ResponseNotChunked(
                 "Response is not chunked. "
-                "Header 'transfer-encoding: chunked' is missing.")
+                "Header 'transfer-encoding: chunked' is missing."
+            )
         if not self.supports_chunked_reads():
             raise BodyNotHttplibCompatible(
                 "Body should be httplib.HTTPResponse like. "
-                "It should have have an fp attribute which returns raw chunks.")
-
-        # Don't bother reading the body of a HEAD request.
-        if self._original_response and is_response_to_head(self._original_response):
-            self._original_response.close()
-            return
+                "It should have have an fp attribute which returns raw chunks."
+            )
 
         with self._error_catcher():
+            # Don't bother reading the body of a HEAD request.
+            if self._original_response and is_response_to_head(self._original_response):
+                self._original_response.close()
+                return
+
+            # If a response is already read and closed
+            # then return immediately.
+            if self._fp.fp is None:
+                return
+
             while True:
                 self._update_chunk_length()
                 if self.chunk_left == 0:
                     break
                 chunk = self._handle_chunk(amt)
-                decoded = self._decode(chunk, decode_content=decode_content,
-                                       flush_decoder=False)
+                decoded = self._decode(
+                    chunk, decode_content=decode_content, flush_decoder=False
+                )
                 if decoded:
                     yield decoded
 
@@ -618,9 +773,37 @@ class HTTPResponse(io.IOBase):
                 if not line:
                     # Some sites may not end with '\r\n'.
                     break
-                if line == b'\r\n':
+                if line == b"\r\n":
                     break
 
             # We read everything; close the "file".
             if self._original_response:
                 self._original_response.close()
+
+    def geturl(self):
+        """
+        Returns the URL that was the source of this response.
+        If the request that generated this response redirected, this method
+        will return the final redirect location.
+        """
+        if self.retries is not None and len(self.retries.history):
+            return self.retries.history[-1].redirect_location
+        else:
+            return self._request_url
+
+    def __iter__(self):
+        buffer = [b""]
+        for chunk in self.stream(decode_content=True):
+            if b"\n" in chunk:
+                chunk = chunk.split(b"\n")
+                yield b"".join(buffer) + chunk[0] + b"\n"
+                for x in chunk[1:-1]:
+                    yield x + b"\n"
+                if chunk[-1]:
+                    buffer = [chunk[-1]]
+                else:
+                    buffer = []
+            else:
+                buffer.append(chunk)
+        if buffer:
+            yield b"".join(buffer)
diff --git a/lib/urllib3/util/__init__.py b/lib/urllib3/util/__init__.py
index 2f2770b6..a96c73a9 100644
--- a/lib/urllib3/util/__init__.py
+++ b/lib/urllib3/util/__init__.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 # For backwards compatibility, provide imports that used to be here.
 from .connection import is_connection_dropped
 from .request import make_headers
@@ -12,43 +13,34 @@ from .ssl_ import (
     resolve_cert_reqs,
     resolve_ssl_version,
     ssl_wrap_socket,
+    PROTOCOL_TLS,
 )
-from .timeout import (
-    current_time,
-    Timeout,
-)
+from .timeout import current_time, Timeout
 
 from .retry import Retry
-from .url import (
-    get_host,
-    parse_url,
-    split_first,
-    Url,
-)
-from .wait import (
-    wait_for_read,
-    wait_for_write
-)
+from .url import get_host, parse_url, split_first, Url
+from .wait import wait_for_read, wait_for_write
 
 __all__ = (
-    'HAS_SNI',
-    'IS_PYOPENSSL',
-    'IS_SECURETRANSPORT',
-    'SSLContext',
-    'Retry',
-    'Timeout',
-    'Url',
-    'assert_fingerprint',
-    'current_time',
-    'is_connection_dropped',
-    'is_fp_closed',
-    'get_host',
-    'parse_url',
-    'make_headers',
-    'resolve_cert_reqs',
-    'resolve_ssl_version',
-    'split_first',
-    'ssl_wrap_socket',
-    'wait_for_read',
-    'wait_for_write'
+    "HAS_SNI",
+    "IS_PYOPENSSL",
+    "IS_SECURETRANSPORT",
+    "SSLContext",
+    "PROTOCOL_TLS",
+    "Retry",
+    "Timeout",
+    "Url",
+    "assert_fingerprint",
+    "current_time",
+    "is_connection_dropped",
+    "is_fp_closed",
+    "get_host",
+    "parse_url",
+    "make_headers",
+    "resolve_cert_reqs",
+    "resolve_ssl_version",
+    "split_first",
+    "ssl_wrap_socket",
+    "wait_for_read",
+    "wait_for_write",
 )
diff --git a/lib/urllib3/util/connection.py b/lib/urllib3/util/connection.py
index bf699cfd..0e111262 100644
--- a/lib/urllib3/util/connection.py
+++ b/lib/urllib3/util/connection.py
@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import socket
-from .wait import wait_for_read
-from .selectors import HAS_SELECT, SelectorError
+from .wait import NoWayToWaitForSocketError, wait_for_read
+from ..contrib import _appengine_environ
 
 
 def is_connection_dropped(conn):  # Platform-specific
@@ -14,27 +14,28 @@ def is_connection_dropped(conn):  # Platform-specific
     Note: For platforms like AppEngine, this will always return ``False`` to
     let the platform handle connection recycling transparently for us.
     """
-    sock = getattr(conn, 'sock', False)
+    sock = getattr(conn, "sock", False)
     if sock is False:  # Platform-specific: AppEngine
         return False
     if sock is None:  # Connection already closed (such as by httplib).
         return True
-
-    if not HAS_SELECT:
-        return False
-
     try:
-        return bool(wait_for_read(sock, timeout=0.0))
-    except SelectorError:
-        return True
+        # Returns True if readable, which here means it's been dropped
+        return wait_for_read(sock, timeout=0.0)
+    except NoWayToWaitForSocketError:  # Platform-specific: AppEngine
+        return False
 
 
 # This function is copied from socket.py in the Python 2.7 standard
 # library test suite. Added to its signature is only `socket_options`.
 # One additional modification is that we avoid binding to IPv6 servers
 # discovered in DNS if the system doesn't have IPv6 functionality.
-def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                      source_address=None, socket_options=None):
+def create_connection(
+    address,
+    timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+    source_address=None,
+    socket_options=None,
+):
     """Connect to *address* and return the socket object.
 
     Convenience function.  Connect to *address* (a 2-tuple ``(host,
@@ -48,8 +49,8 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
     """
 
     host, port = address
-    if host.startswith('['):
-        host = host.strip('[]')
+    if host.startswith("["):
+        host = host.strip("[]")
     err = None
 
     # Using the value from allowed_gai_family() in the context of getaddrinfo lets
@@ -109,6 +110,13 @@ def _has_ipv6(host):
     sock = None
     has_ipv6 = False
 
+    # App Engine doesn't support IPV6 sockets and actually has a quota on the
+    # number of sockets that can be used, so just early out here instead of
+    # creating a socket needlessly.
+    # See https://github.com/urllib3/urllib3/issues/1446
+    if _appengine_environ.is_appengine_sandbox():
+        return False
+
     if socket.has_ipv6:
         # has_ipv6 returns true if cPython was compiled with IPv6 support.
         # It does not tell us if the system has IPv6 support enabled. To
@@ -127,4 +135,4 @@ def _has_ipv6(host):
     return has_ipv6
 
 
-HAS_IPV6 = _has_ipv6('::1')
+HAS_IPV6 = _has_ipv6("::1")
diff --git a/lib/urllib3/util/queue.py b/lib/urllib3/util/queue.py
new file mode 100644
index 00000000..d3d379a1
--- /dev/null
+++ b/lib/urllib3/util/queue.py
@@ -0,0 +1,21 @@
+import collections
+from ..packages import six
+from ..packages.six.moves import queue
+
+if six.PY2:
+    # Queue is imported for side effects on MS Windows. See issue #229.
+    import Queue as _unused_module_Queue  # noqa: F401
+
+
+class LifoQueue(queue.Queue):
+    def _init(self, _):
+        self.queue = collections.deque()
+
+    def _qsize(self, len=len):
+        return len(self.queue)
+
+    def _put(self, item):
+        self.queue.append(item)
+
+    def _get(self):
+        return self.queue.pop()
diff --git a/lib/urllib3/util/request.py b/lib/urllib3/util/request.py
index 3ddfcd55..262a6d61 100644
--- a/lib/urllib3/util/request.py
+++ b/lib/urllib3/util/request.py
@@ -4,12 +4,25 @@ from base64 import b64encode
 from ..packages.six import b, integer_types
 from ..exceptions import UnrewindableBodyError
 
-ACCEPT_ENCODING = 'gzip,deflate'
+ACCEPT_ENCODING = "gzip,deflate"
+try:
+    import brotli as _unused_module_brotli  # noqa: F401
+except ImportError:
+    pass
+else:
+    ACCEPT_ENCODING += ",br"
+
 _FAILEDTELL = object()
 
 
-def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
-                 basic_auth=None, proxy_basic_auth=None, disable_cache=None):
+def make_headers(
+    keep_alive=None,
+    accept_encoding=None,
+    user_agent=None,
+    basic_auth=None,
+    proxy_basic_auth=None,
+    disable_cache=None,
+):
     """
     Shortcuts for generating request headers.
 
@@ -49,27 +62,27 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
         if isinstance(accept_encoding, str):
             pass
         elif isinstance(accept_encoding, list):
-            accept_encoding = ','.join(accept_encoding)
+            accept_encoding = ",".join(accept_encoding)
         else:
             accept_encoding = ACCEPT_ENCODING
-        headers['accept-encoding'] = accept_encoding
+        headers["accept-encoding"] = accept_encoding
 
     if user_agent:
-        headers['user-agent'] = user_agent
+        headers["user-agent"] = user_agent
 
     if keep_alive:
-        headers['connection'] = 'keep-alive'
+        headers["connection"] = "keep-alive"
 
     if basic_auth:
-        headers['authorization'] = 'Basic ' + \
-            b64encode(b(basic_auth)).decode('utf-8')
+        headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8")
 
     if proxy_basic_auth:
-        headers['proxy-authorization'] = 'Basic ' + \
-            b64encode(b(proxy_basic_auth)).decode('utf-8')
+        headers["proxy-authorization"] = "Basic " + b64encode(
+            b(proxy_basic_auth)
+        ).decode("utf-8")
 
     if disable_cache:
-        headers['cache-control'] = 'no-cache'
+        headers["cache-control"] = "no-cache"
 
     return headers
 
@@ -81,7 +94,7 @@ def set_file_position(body, pos):
     """
     if pos is not None:
         rewind_body(body, pos)
-    elif getattr(body, 'tell', None) is not None:
+    elif getattr(body, "tell", None) is not None:
         try:
             pos = body.tell()
         except (IOError, OSError):
@@ -103,16 +116,20 @@ def rewind_body(body, body_pos):
     :param int pos:
         Position to seek to in file.
     """
-    body_seek = getattr(body, 'seek', None)
+    body_seek = getattr(body, "seek", None)
     if body_seek is not None and isinstance(body_pos, integer_types):
         try:
             body_seek(body_pos)
         except (IOError, OSError):
-            raise UnrewindableBodyError("An error occurred when rewinding request "
-                                        "body for redirect/retry.")
+            raise UnrewindableBodyError(
+                "An error occurred when rewinding request " "body for redirect/retry."
+            )
     elif body_pos is _FAILEDTELL:
-        raise UnrewindableBodyError("Unable to record file position for rewinding "
-                                    "request body during a redirect/retry.")
+        raise UnrewindableBodyError(
+            "Unable to record file position for rewinding "
+            "request body during a redirect/retry."
+        )
     else:
-        raise ValueError("body_pos must be of type integer, "
-                         "instead it was %s." % type(body_pos))
+        raise ValueError(
+            "body_pos must be of type integer, " "instead it was %s." % type(body_pos)
+        )
diff --git a/lib/urllib3/util/response.py b/lib/urllib3/util/response.py
index 67cf730a..715868dd 100644
--- a/lib/urllib3/util/response.py
+++ b/lib/urllib3/util/response.py
@@ -52,15 +52,20 @@ def assert_header_parsing(headers):
     # This will fail silently if we pass in the wrong kind of parameter.
     # To make debugging easier add an explicit check.
     if not isinstance(headers, httplib.HTTPMessage):
-        raise TypeError('expected httplib.Message, got {0}.'.format(
-            type(headers)))
+        raise TypeError("expected httplib.Message, got {0}.".format(type(headers)))
 
-    defects = getattr(headers, 'defects', None)
-    get_payload = getattr(headers, 'get_payload', None)
+    defects = getattr(headers, "defects", None)
+    get_payload = getattr(headers, "get_payload", None)
 
     unparsed_data = None
-    if get_payload:  # Platform-specific: Python 3.
-        unparsed_data = get_payload()
+    if get_payload:
+        # get_payload is actually email.message.Message.get_payload;
+        # we're only interested in the result if it's not a multipart message
+        if not headers.is_multipart():
+            payload = get_payload()
+
+            if isinstance(payload, (bytes, str)):
+                unparsed_data = payload
 
     if defects or unparsed_data:
         raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
@@ -78,4 +83,4 @@ def is_response_to_head(response):
     method = response._method
     if isinstance(method, int):  # Platform-specific: Appengine
         return method == 3
-    return method.upper() == 'HEAD'
+    return method.upper() == "HEAD"
diff --git a/lib/urllib3/util/retry.py b/lib/urllib3/util/retry.py
index c603cb49..5a049fe6 100644
--- a/lib/urllib3/util/retry.py
+++ b/lib/urllib3/util/retry.py
@@ -19,9 +19,11 @@ from ..packages import six
 
 log = logging.getLogger(__name__)
 
+
 # Data structure for representing the metadata of requests that result in a retry.
-RequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
-                                               "status", "redirect_location"])
+RequestHistory = namedtuple(
+    "RequestHistory", ["method", "url", "error", "status", "redirect_location"]
+)
 
 
 class Retry(object):
@@ -114,7 +116,7 @@ class Retry(object):
         (most errors are resolved immediately by a second try without a
         delay). urllib3 will sleep for::
 
-            {backoff factor} * (2 ^ ({number of total retries} - 1))
+            {backoff factor} * (2 ** ({number of total retries} - 1))
 
         seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
         for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
@@ -139,20 +141,39 @@ class Retry(object):
         Whether to respect Retry-After header on status codes defined as
         :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
 
+    :param iterable remove_headers_on_redirect:
+        Sequence of headers to remove from the request when a response
+        indicating a redirect is returned before firing off the redirected
+        request.
     """
 
-    DEFAULT_METHOD_WHITELIST = frozenset([
-        'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
+    DEFAULT_METHOD_WHITELIST = frozenset(
+        ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
+    )
 
     RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
 
+    DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"])
+
     #: Maximum backoff time.
     BACKOFF_MAX = 120
 
-    def __init__(self, total=10, connect=None, read=None, redirect=None, status=None,
-                 method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
-                 backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
-                 history=None, respect_retry_after_header=True):
+    def __init__(
+        self,
+        total=10,
+        connect=None,
+        read=None,
+        redirect=None,
+        status=None,
+        method_whitelist=DEFAULT_METHOD_WHITELIST,
+        status_forcelist=None,
+        backoff_factor=0,
+        raise_on_redirect=True,
+        raise_on_status=True,
+        history=None,
+        respect_retry_after_header=True,
+        remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST,
+    ):
 
         self.total = total
         self.connect = connect
@@ -171,17 +192,25 @@ class Retry(object):
         self.raise_on_status = raise_on_status
         self.history = history or tuple()
         self.respect_retry_after_header = respect_retry_after_header
+        self.remove_headers_on_redirect = frozenset(
+            [h.lower() for h in remove_headers_on_redirect]
+        )
 
     def new(self, **kw):
         params = dict(
             total=self.total,
-            connect=self.connect, read=self.read, redirect=self.redirect, status=self.status,
+            connect=self.connect,
+            read=self.read,
+            redirect=self.redirect,
+            status=self.status,
             method_whitelist=self.method_whitelist,
             status_forcelist=self.status_forcelist,
             backoff_factor=self.backoff_factor,
             raise_on_redirect=self.raise_on_redirect,
             raise_on_status=self.raise_on_status,
             history=self.history,
+            remove_headers_on_redirect=self.remove_headers_on_redirect,
+            respect_retry_after_header=self.respect_retry_after_header,
         )
         params.update(kw)
         return type(self)(**params)
@@ -206,8 +235,11 @@ class Retry(object):
         :rtype: float
         """
         # We want to consider only the last consecutive errors sequence (Ignore redirects).
-        consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
-                                                    reversed(self.history))))
+        consecutive_errors_len = len(
+            list(
+                takewhile(lambda x: x.redirect_location is None, reversed(self.history))
+            )
+        )
         if consecutive_errors_len <= 1:
             return 0
 
@@ -263,7 +295,7 @@ class Retry(object):
         this method will return immediately.
         """
 
-        if response:
+        if self.respect_retry_after_header and response:
             slept = self.sleep_for_retry(response)
             if slept:
                 return
@@ -304,8 +336,12 @@ class Retry(object):
         if self.status_forcelist and status_code in self.status_forcelist:
             return True
 
-        return (self.total and self.respect_retry_after_header and
-                has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
+        return (
+            self.total
+            and self.respect_retry_after_header
+            and has_retry_after
+            and (status_code in self.RETRY_AFTER_STATUS_CODES)
+        )
 
     def is_exhausted(self):
         """ Are we out of retries? """
@@ -316,8 +352,15 @@ class Retry(object):
 
         return min(retry_counts) < 0
 
-    def increment(self, method=None, url=None, response=None, error=None,
-                  _pool=None, _stacktrace=None):
+    def increment(
+        self,
+        method=None,
+        url=None,
+        response=None,
+        error=None,
+        _pool=None,
+        _stacktrace=None,
+    ):
         """ Return a new Retry object with incremented retry counters.
 
         :param response: A response object, or None, if the server did not
@@ -340,7 +383,7 @@ class Retry(object):
         read = self.read
         redirect = self.redirect
         status_count = self.status
-        cause = 'unknown'
+        cause = "unknown"
         status = None
         redirect_location = None
 
@@ -362,7 +405,7 @@ class Retry(object):
             # Redirect retry?
             if redirect is not None:
                 redirect -= 1
-            cause = 'too many redirects'
+            cause = "too many redirects"
             redirect_location = response.get_redirect_location()
             status = response.status
 
@@ -373,16 +416,21 @@ class Retry(object):
             if response and response.status:
                 if status_count is not None:
                     status_count -= 1
-                cause = ResponseError.SPECIFIC_ERROR.format(
-                    status_code=response.status)
+                cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
                 status = response.status
 
-        history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
+        history = self.history + (
+            RequestHistory(method, url, error, status, redirect_location),
+        )
 
         new_retry = self.new(
             total=total,
-            connect=connect, read=read, redirect=redirect, status=status_count,
-            history=history)
+            connect=connect,
+            read=read,
+            redirect=redirect,
+            status=status_count,
+            history=history,
+        )
 
         if new_retry.is_exhausted():
             raise MaxRetryError(_pool, url, error or ResponseError(cause))
@@ -392,9 +440,10 @@ class Retry(object):
         return new_retry
 
     def __repr__(self):
-        return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
-                'read={self.read}, redirect={self.redirect}, status={self.status})').format(
-                    cls=type(self), self=self)
+        return (
+            "{cls.__name__}(total={self.total}, connect={self.connect}, "
+            "read={self.read}, redirect={self.redirect}, status={self.status})"
+        ).format(cls=type(self), self=self)
 
 
 # For backwards compatibility (equivalent to pre-v1.9):
diff --git a/lib/urllib3/util/selectors.py b/lib/urllib3/util/selectors.py
deleted file mode 100644
index d75cb266..00000000
--- a/lib/urllib3/util/selectors.py
+++ /dev/null
@@ -1,581 +0,0 @@
-# Backport of selectors.py from Python 3.5+ to support Python < 3.4
-# Also has the behavior specified in PEP 475 which is to retry syscalls
-# in the case of an EINTR error. This module is required because selectors34
-# does not follow this behavior and instead returns that no dile descriptor
-# events have occurred rather than retry the syscall. The decision to drop
-# support for select.devpoll is made to maintain 100% test coverage.
-
-import errno
-import math
-import select
-import socket
-import sys
-import time
-from collections import namedtuple, Mapping
-
-try:
-    monotonic = time.monotonic
-except (AttributeError, ImportError):  # Python 3.3<
-    monotonic = time.time
-
-EVENT_READ = (1 << 0)
-EVENT_WRITE = (1 << 1)
-
-HAS_SELECT = True  # Variable that shows whether the platform has a selector.
-_SYSCALL_SENTINEL = object()  # Sentinel in case a system call returns None.
-_DEFAULT_SELECTOR = None
-
-
-class SelectorError(Exception):
-    def __init__(self, errcode):
-        super(SelectorError, self).__init__()
-        self.errno = errcode
-
-    def __repr__(self):
-        return "".format(self.errno)
-
-    def __str__(self):
-        return self.__repr__()
-
-
-def _fileobj_to_fd(fileobj):
-    """ Return a file descriptor from a file object. If
-    given an integer will simply return that integer back. """
-    if isinstance(fileobj, int):
-        fd = fileobj
-    else:
-        try:
-            fd = int(fileobj.fileno())
-        except (AttributeError, TypeError, ValueError):
-            raise ValueError("Invalid file object: {0!r}".format(fileobj))
-    if fd < 0:
-        raise ValueError("Invalid file descriptor: {0}".format(fd))
-    return fd
-
-
-# Determine which function to use to wrap system calls because Python 3.5+
-# already handles the case when system calls are interrupted.
-if sys.version_info >= (3, 5):
-    def _syscall_wrapper(func, _, *args, **kwargs):
-        """ This is the short-circuit version of the below logic
-        because in Python 3.5+ all system calls automatically restart
-        and recalculate their timeouts. """
-        try:
-            return func(*args, **kwargs)
-        except (OSError, IOError, select.error) as e:
-            errcode = None
-            if hasattr(e, "errno"):
-                errcode = e.errno
-            raise SelectorError(errcode)
-else:
-    def _syscall_wrapper(func, recalc_timeout, *args, **kwargs):
-        """ Wrapper function for syscalls that could fail due to EINTR.
-        All functions should be retried if there is time left in the timeout
-        in accordance with PEP 475. """
-        timeout = kwargs.get("timeout", None)
-        if timeout is None:
-            expires = None
-            recalc_timeout = False
-        else:
-            timeout = float(timeout)
-            if timeout < 0.0:  # Timeout less than 0 treated as no timeout.
-                expires = None
-            else:
-                expires = monotonic() + timeout
-
-        args = list(args)
-        if recalc_timeout and "timeout" not in kwargs:
-            raise ValueError(
-                "Timeout must be in args or kwargs to be recalculated")
-
-        result = _SYSCALL_SENTINEL
-        while result is _SYSCALL_SENTINEL:
-            try:
-                result = func(*args, **kwargs)
-            # OSError is thrown by select.select
-            # IOError is thrown by select.epoll.poll
-            # select.error is thrown by select.poll.poll
-            # Aren't we thankful for Python 3.x rework for exceptions?
-            except (OSError, IOError, select.error) as e:
-                # select.error wasn't a subclass of OSError in the past.
-                errcode = None
-                if hasattr(e, "errno"):
-                    errcode = e.errno
-                elif hasattr(e, "args"):
-                    errcode = e.args[0]
-
-                # Also test for the Windows equivalent of EINTR.
-                is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and
-                                                           errcode == errno.WSAEINTR))
-
-                if is_interrupt:
-                    if expires is not None:
-                        current_time = monotonic()
-                        if current_time > expires:
-                            raise OSError(errno=errno.ETIMEDOUT)
-                        if recalc_timeout:
-                            if "timeout" in kwargs:
-                                kwargs["timeout"] = expires - current_time
-                    continue
-                if errcode:
-                    raise SelectorError(errcode)
-                else:
-                    raise
-        return result
-
-
-SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
-
-
-class _SelectorMapping(Mapping):
-    """ Mapping of file objects to selector keys """
-
-    def __init__(self, selector):
-        self._selector = selector
-
-    def __len__(self):
-        return len(self._selector._fd_to_key)
-
-    def __getitem__(self, fileobj):
-        try:
-            fd = self._selector._fileobj_lookup(fileobj)
-            return self._selector._fd_to_key[fd]
-        except KeyError:
-            raise KeyError("{0!r} is not registered.".format(fileobj))
-
-    def __iter__(self):
-        return iter(self._selector._fd_to_key)
-
-
-class BaseSelector(object):
-    """ Abstract Selector class
-
-    A selector supports registering file objects to be monitored
-    for specific I/O events.
-
-    A file object is a file descriptor or any object with a
-    `fileno()` method. An arbitrary object can be attached to the
-    file object which can be used for example to store context info,
-    a callback, etc.
-
-    A selector can use various implementations (select(), poll(), epoll(),
-    and kqueue()) depending on the platform. The 'DefaultSelector' class uses
-    the most efficient implementation for the current platform.
-    """
-    def __init__(self):
-        # Maps file descriptors to keys.
-        self._fd_to_key = {}
-
-        # Read-only mapping returned by get_map()
-        self._map = _SelectorMapping(self)
-
-    def _fileobj_lookup(self, fileobj):
-        """ Return a file descriptor from a file object.
-        This wraps _fileobj_to_fd() to do an exhaustive
-        search in case the object is invalid but we still
-        have it in our map. Used by unregister() so we can
-        unregister an object that was previously registered
-        even if it is closed. It is also used by _SelectorMapping
-        """
-        try:
-            return _fileobj_to_fd(fileobj)
-        except ValueError:
-
-            # Search through all our mapped keys.
-            for key in self._fd_to_key.values():
-                if key.fileobj is fileobj:
-                    return key.fd
-
-            # Raise ValueError after all.
-            raise
-
-    def register(self, fileobj, events, data=None):
-        """ Register a file object for a set of events to monitor. """
-        if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
-            raise ValueError("Invalid events: {0!r}".format(events))
-
-        key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
-
-        if key.fd in self._fd_to_key:
-            raise KeyError("{0!r} (FD {1}) is already registered"
-                           .format(fileobj, key.fd))
-
-        self._fd_to_key[key.fd] = key
-        return key
-
-    def unregister(self, fileobj):
-        """ Unregister a file object from being monitored. """
-        try:
-            key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
-        except KeyError:
-            raise KeyError("{0!r} is not registered".format(fileobj))
-
-        # Getting the fileno of a closed socket on Windows errors with EBADF.
-        except socket.error as e:  # Platform-specific: Windows.
-            if e.errno != errno.EBADF:
-                raise
-            else:
-                for key in self._fd_to_key.values():
-                    if key.fileobj is fileobj:
-                        self._fd_to_key.pop(key.fd)
-                        break
-                else:
-                    raise KeyError("{0!r} is not registered".format(fileobj))
-        return key
-
-    def modify(self, fileobj, events, data=None):
-        """ Change a registered file object monitored events and data. """
-        # NOTE: Some subclasses optimize this operation even further.
-        try:
-            key = self._fd_to_key[self._fileobj_lookup(fileobj)]
-        except KeyError:
-            raise KeyError("{0!r} is not registered".format(fileobj))
-
-        if events != key.events:
-            self.unregister(fileobj)
-            key = self.register(fileobj, events, data)
-
-        elif data != key.data:
-            # Use a shortcut to update the data.
-            key = key._replace(data=data)
-            self._fd_to_key[key.fd] = key
-
-        return key
-
-    def select(self, timeout=None):
-        """ Perform the actual selection until some monitored file objects
-        are ready or the timeout expires. """
-        raise NotImplementedError()
-
-    def close(self):
-        """ Close the selector. This must be called to ensure that all
-        underlying resources are freed. """
-        self._fd_to_key.clear()
-        self._map = None
-
-    def get_key(self, fileobj):
-        """ Return the key associated with a registered file object. """
-        mapping = self.get_map()
-        if mapping is None:
-            raise RuntimeError("Selector is closed")
-        try:
-            return mapping[fileobj]
-        except KeyError:
-            raise KeyError("{0!r} is not registered".format(fileobj))
-
-    def get_map(self):
-        """ Return a mapping of file objects to selector keys """
-        return self._map
-
-    def _key_from_fd(self, fd):
-        """ Return the key associated to a given file descriptor
-         Return None if it is not found. """
-        try:
-            return self._fd_to_key[fd]
-        except KeyError:
-            return None
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-
-# Almost all platforms have select.select()
-if hasattr(select, "select"):
-    class SelectSelector(BaseSelector):
-        """ Select-based selector. """
-        def __init__(self):
-            super(SelectSelector, self).__init__()
-            self._readers = set()
-            self._writers = set()
-
-        def register(self, fileobj, events, data=None):
-            key = super(SelectSelector, self).register(fileobj, events, data)
-            if events & EVENT_READ:
-                self._readers.add(key.fd)
-            if events & EVENT_WRITE:
-                self._writers.add(key.fd)
-            return key
-
-        def unregister(self, fileobj):
-            key = super(SelectSelector, self).unregister(fileobj)
-            self._readers.discard(key.fd)
-            self._writers.discard(key.fd)
-            return key
-
-        def _select(self, r, w, timeout=None):
-            """ Wrapper for select.select because timeout is a positional arg """
-            return select.select(r, w, [], timeout)
-
-        def select(self, timeout=None):
-            # Selecting on empty lists on Windows errors out.
-            if not len(self._readers) and not len(self._writers):
-                return []
-
-            timeout = None if timeout is None else max(timeout, 0.0)
-            ready = []
-            r, w, _ = _syscall_wrapper(self._select, True, self._readers,
-                                       self._writers, timeout)
-            r = set(r)
-            w = set(w)
-            for fd in r | w:
-                events = 0
-                if fd in r:
-                    events |= EVENT_READ
-                if fd in w:
-                    events |= EVENT_WRITE
-
-                key = self._key_from_fd(fd)
-                if key:
-                    ready.append((key, events & key.events))
-            return ready
-
-
-if hasattr(select, "poll"):
-    class PollSelector(BaseSelector):
-        """ Poll-based selector """
-        def __init__(self):
-            super(PollSelector, self).__init__()
-            self._poll = select.poll()
-
-        def register(self, fileobj, events, data=None):
-            key = super(PollSelector, self).register(fileobj, events, data)
-            event_mask = 0
-            if events & EVENT_READ:
-                event_mask |= select.POLLIN
-            if events & EVENT_WRITE:
-                event_mask |= select.POLLOUT
-            self._poll.register(key.fd, event_mask)
-            return key
-
-        def unregister(self, fileobj):
-            key = super(PollSelector, self).unregister(fileobj)
-            self._poll.unregister(key.fd)
-            return key
-
-        def _wrap_poll(self, timeout=None):
-            """ Wrapper function for select.poll.poll() so that
-            _syscall_wrapper can work with only seconds. """
-            if timeout is not None:
-                if timeout <= 0:
-                    timeout = 0
-                else:
-                    # select.poll.poll() has a resolution of 1 millisecond,
-                    # round away from zero to wait *at least* timeout seconds.
-                    timeout = math.ceil(timeout * 1e3)
-
-            result = self._poll.poll(timeout)
-            return result
-
-        def select(self, timeout=None):
-            ready = []
-            fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout)
-            for fd, event_mask in fd_events:
-                events = 0
-                if event_mask & ~select.POLLIN:
-                    events |= EVENT_WRITE
-                if event_mask & ~select.POLLOUT:
-                    events |= EVENT_READ
-
-                key = self._key_from_fd(fd)
-                if key:
-                    ready.append((key, events & key.events))
-
-            return ready
-
-
-if hasattr(select, "epoll"):
-    class EpollSelector(BaseSelector):
-        """ Epoll-based selector """
-        def __init__(self):
-            super(EpollSelector, self).__init__()
-            self._epoll = select.epoll()
-
-        def fileno(self):
-            return self._epoll.fileno()
-
-        def register(self, fileobj, events, data=None):
-            key = super(EpollSelector, self).register(fileobj, events, data)
-            events_mask = 0
-            if events & EVENT_READ:
-                events_mask |= select.EPOLLIN
-            if events & EVENT_WRITE:
-                events_mask |= select.EPOLLOUT
-            _syscall_wrapper(self._epoll.register, False, key.fd, events_mask)
-            return key
-
-        def unregister(self, fileobj):
-            key = super(EpollSelector, self).unregister(fileobj)
-            try:
-                _syscall_wrapper(self._epoll.unregister, False, key.fd)
-            except SelectorError:
-                # This can occur when the fd was closed since registry.
-                pass
-            return key
-
-        def select(self, timeout=None):
-            if timeout is not None:
-                if timeout <= 0:
-                    timeout = 0.0
-                else:
-                    # select.epoll.poll() has a resolution of 1 millisecond
-                    # but luckily takes seconds so we don't need a wrapper
-                    # like PollSelector. Just for better rounding.
-                    timeout = math.ceil(timeout * 1e3) * 1e-3
-                timeout = float(timeout)
-            else:
-                timeout = -1.0  # epoll.poll() must have a float.
-
-            # We always want at least 1 to ensure that select can be called
-            # with no file descriptors registered. Otherwise will fail.
-            max_events = max(len(self._fd_to_key), 1)
-
-            ready = []
-            fd_events = _syscall_wrapper(self._epoll.poll, True,
-                                         timeout=timeout,
-                                         maxevents=max_events)
-            for fd, event_mask in fd_events:
-                events = 0
-                if event_mask & ~select.EPOLLIN:
-                    events |= EVENT_WRITE
-                if event_mask & ~select.EPOLLOUT:
-                    events |= EVENT_READ
-
-                key = self._key_from_fd(fd)
-                if key:
-                    ready.append((key, events & key.events))
-            return ready
-
-        def close(self):
-            self._epoll.close()
-            super(EpollSelector, self).close()
-
-
-if hasattr(select, "kqueue"):
-    class KqueueSelector(BaseSelector):
-        """ Kqueue / Kevent-based selector """
-        def __init__(self):
-            super(KqueueSelector, self).__init__()
-            self._kqueue = select.kqueue()
-
-        def fileno(self):
-            return self._kqueue.fileno()
-
-        def register(self, fileobj, events, data=None):
-            key = super(KqueueSelector, self).register(fileobj, events, data)
-            if events & EVENT_READ:
-                kevent = select.kevent(key.fd,
-                                       select.KQ_FILTER_READ,
-                                       select.KQ_EV_ADD)
-
-                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
-
-            if events & EVENT_WRITE:
-                kevent = select.kevent(key.fd,
-                                       select.KQ_FILTER_WRITE,
-                                       select.KQ_EV_ADD)
-
-                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
-
-            return key
-
-        def unregister(self, fileobj):
-            key = super(KqueueSelector, self).unregister(fileobj)
-            if key.events & EVENT_READ:
-                kevent = select.kevent(key.fd,
-                                       select.KQ_FILTER_READ,
-                                       select.KQ_EV_DELETE)
-                try:
-                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
-                except SelectorError:
-                    pass
-            if key.events & EVENT_WRITE:
-                kevent = select.kevent(key.fd,
-                                       select.KQ_FILTER_WRITE,
-                                       select.KQ_EV_DELETE)
-                try:
-                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
-                except SelectorError:
-                    pass
-
-            return key
-
-        def select(self, timeout=None):
-            if timeout is not None:
-                timeout = max(timeout, 0)
-
-            max_events = len(self._fd_to_key) * 2
-            ready_fds = {}
-
-            kevent_list = _syscall_wrapper(self._kqueue.control, True,
-                                           None, max_events, timeout)
-
-            for kevent in kevent_list:
-                fd = kevent.ident
-                event_mask = kevent.filter
-                events = 0
-                if event_mask == select.KQ_FILTER_READ:
-                    events |= EVENT_READ
-                if event_mask == select.KQ_FILTER_WRITE:
-                    events |= EVENT_WRITE
-
-                key = self._key_from_fd(fd)
-                if key:
-                    if key.fd not in ready_fds:
-                        ready_fds[key.fd] = (key, events & key.events)
-                    else:
-                        old_events = ready_fds[key.fd][1]
-                        ready_fds[key.fd] = (key, (events | old_events) & key.events)
-
-            return list(ready_fds.values())
-
-        def close(self):
-            self._kqueue.close()
-            super(KqueueSelector, self).close()
-
-
-if not hasattr(select, 'select'):  # Platform-specific: AppEngine
-    HAS_SELECT = False
-
-
-def _can_allocate(struct):
-    """ Checks that select structs can be allocated by the underlying
-    operating system, not just advertised by the select module. We don't
-    check select() because we'll be hopeful that most platforms that
-    don't have it available will not advertise it. (ie: GAE) """
-    try:
-        # select.poll() objects won't fail until used.
-        if struct == 'poll':
-            p = select.poll()
-            p.poll(0)
-
-        # All others will fail on allocation.
-        else:
-            getattr(select, struct)().close()
-        return True
-    except (OSError, AttributeError) as e:
-        return False
-
-
-# Choose the best implementation, roughly:
-# kqueue == epoll > poll > select. Devpoll not supported. (See above)
-# select() also can't accept a FD > FD_SETSIZE (usually around 1024)
-def DefaultSelector():
-    """ This function serves as a first call for DefaultSelector to
-    detect if the select module is being monkey-patched incorrectly
-    by eventlet, greenlet, and preserve proper behavior. """
-    global _DEFAULT_SELECTOR
-    if _DEFAULT_SELECTOR is None:
-        if _can_allocate('kqueue'):
-            _DEFAULT_SELECTOR = KqueueSelector
-        elif _can_allocate('epoll'):
-            _DEFAULT_SELECTOR = EpollSelector
-        elif _can_allocate('poll'):
-            _DEFAULT_SELECTOR = PollSelector
-        elif hasattr(select, 'select'):
-            _DEFAULT_SELECTOR = SelectSelector
-        else:  # Platform-specific: AppEngine
-            raise ValueError('Platform does not have a selector')
-    return _DEFAULT_SELECTOR()
diff --git a/lib/urllib3/util/ssl_.py b/lib/urllib3/util/ssl_.py
index 32fd9eda..8495b775 100644
--- a/lib/urllib3/util/ssl_.py
+++ b/lib/urllib3/util/ssl_.py
@@ -2,11 +2,14 @@ from __future__ import absolute_import
 import errno
 import warnings
 import hmac
+import sys
 
 from binascii import hexlify, unhexlify
 from hashlib import md5, sha1, sha256
 
+from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE
 from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
+from ..packages import six
 
 
 SSLContext = None
@@ -15,11 +18,7 @@ IS_PYOPENSSL = False
 IS_SECURETRANSPORT = False
 
 # Maps the length of a digest to a possible hash function producing this digest
-HASHFUNC_MAP = {
-    32: md5,
-    40: sha1,
-    64: sha256,
-}
+HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256}
 
 
 def _const_compare_digest_backport(a, b):
@@ -35,17 +34,27 @@ def _const_compare_digest_backport(a, b):
     return result == 0
 
 
-_const_compare_digest = getattr(hmac, 'compare_digest',
-                                _const_compare_digest_backport)
-
+_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport)
 
 try:  # Test for SSL features
     import ssl
-    from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
+    from ssl import wrap_socket, CERT_REQUIRED
     from ssl import HAS_SNI  # Has SNI?
 except ImportError:
     pass
 
+try:  # Platform-specific: Python 3.6
+    from ssl import PROTOCOL_TLS
+
+    PROTOCOL_SSLv23 = PROTOCOL_TLS
+except ImportError:
+    try:
+        from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
+
+        PROTOCOL_SSLv23 = PROTOCOL_TLS
+    except ImportError:
+        PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
+
 
 try:
     from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
@@ -53,6 +62,7 @@ except ImportError:
     OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
     OP_NO_COMPRESSION = 0x20000
 
+
 # A secure default.
 # Sources for more information on TLS ciphers:
 #
@@ -61,41 +71,39 @@ except ImportError:
 # - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
 #
 # The general intent is:
-# - Prefer TLS 1.3 cipher suites
 # - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
 # - prefer ECDHE over DHE for better performance,
 # - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and
 #   security,
 # - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common,
-# - disable NULL authentication, MD5 MACs and DSS for security reasons.
-DEFAULT_CIPHERS = ':'.join([
-    'TLS13-AES-256-GCM-SHA384',
-    'TLS13-CHACHA20-POLY1305-SHA256',
-    'TLS13-AES-128-GCM-SHA256',
-    'ECDH+AESGCM',
-    'ECDH+CHACHA20',
-    'DH+AESGCM',
-    'DH+CHACHA20',
-    'ECDH+AES256',
-    'DH+AES256',
-    'ECDH+AES128',
-    'DH+AES',
-    'RSA+AESGCM',
-    'RSA+AES',
-    '!aNULL',
-    '!eNULL',
-    '!MD5',
-])
+# - disable NULL authentication, MD5 MACs, DSS, and other
+#   insecure ciphers for security reasons.
+# - NOTE: TLS 1.3 cipher suites are managed through a different interface
+#   not exposed by CPython (yet!) and are enabled by default if they're available.
+DEFAULT_CIPHERS = ":".join(
+    [
+        "ECDHE+AESGCM",
+        "ECDHE+CHACHA20",
+        "DHE+AESGCM",
+        "DHE+CHACHA20",
+        "ECDH+AESGCM",
+        "DH+AESGCM",
+        "ECDH+AES",
+        "DH+AES",
+        "RSA+AESGCM",
+        "RSA+AES",
+        "!aNULL",
+        "!eNULL",
+        "!MD5",
+        "!DSS",
+    ]
+)
 
 try:
     from ssl import SSLContext  # Modern SSL?
 except ImportError:
-    import sys
-
-    class SSLContext(object):  # Platform-specific: Python 2 & 3.1
-        supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or
-                                (3, 2) <= sys.version_info)
 
+    class SSLContext(object):  # Platform-specific: Python 2
         def __init__(self, protocol_version):
             self.protocol = protocol_version
             # Use default values from a real SSLContext
@@ -118,36 +126,27 @@ except ImportError:
                 raise SSLError("CA directories not supported in older Pythons")
 
         def set_ciphers(self, cipher_suite):
-            if not self.supports_set_ciphers:
-                raise TypeError(
-                    'Your version of Python does not support setting '
-                    'a custom cipher suite. Please upgrade to Python '
-                    '2.7, 3.2, or later if you need this functionality.'
-                )
             self.ciphers = cipher_suite
 
         def wrap_socket(self, socket, server_hostname=None, server_side=False):
             warnings.warn(
-                'A true SSLContext object is not available. This prevents '
-                'urllib3 from configuring SSL appropriately and may cause '
-                'certain SSL connections to fail. You can upgrade to a newer '
-                'version of Python to solve this. For more information, see '
-                'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
-                '#ssl-warnings',
-                InsecurePlatformWarning
+                "A true SSLContext object is not available. This prevents "
+                "urllib3 from configuring SSL appropriately and may cause "
+                "certain SSL connections to fail. You can upgrade to a newer "
+                "version of Python to solve this. For more information, see "
+                "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
+                "#ssl-warnings",
+                InsecurePlatformWarning,
             )
             kwargs = {
-                'keyfile': self.keyfile,
-                'certfile': self.certfile,
-                'ca_certs': self.ca_certs,
-                'cert_reqs': self.verify_mode,
-                'ssl_version': self.protocol,
-                'server_side': server_side,
+                "keyfile": self.keyfile,
+                "certfile": self.certfile,
+                "ca_certs": self.ca_certs,
+                "cert_reqs": self.verify_mode,
+                "ssl_version": self.protocol,
+                "server_side": server_side,
             }
-            if self.supports_set_ciphers:  # Platform-specific: Python 2.7+
-                return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
-            else:  # Platform-specific: Python 2.6
-                return wrap_socket(socket, **kwargs)
+            return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
 
 
 def assert_fingerprint(cert, fingerprint):
@@ -160,12 +159,11 @@ def assert_fingerprint(cert, fingerprint):
         Fingerprint as string of hexdigits, can be interspersed by colons.
     """
 
-    fingerprint = fingerprint.replace(':', '').lower()
+    fingerprint = fingerprint.replace(":", "").lower()
     digest_length = len(fingerprint)
     hashfunc = HASHFUNC_MAP.get(digest_length)
     if not hashfunc:
-        raise SSLError(
-            'Fingerprint of invalid length: {0}'.format(fingerprint))
+        raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint))
 
     # We need encode() here for py32; works on py2 and p33.
     fingerprint_bytes = unhexlify(fingerprint.encode())
@@ -173,8 +171,11 @@ def assert_fingerprint(cert, fingerprint):
     cert_digest = hashfunc(cert).digest()
 
     if not _const_compare_digest(cert_digest, fingerprint_bytes):
-        raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
-                       .format(fingerprint, hexlify(cert_digest)))
+        raise SSLError(
+            'Fingerprints did not match. Expected "{0}", got "{1}".'.format(
+                fingerprint, hexlify(cert_digest)
+            )
+        )
 
 
 def resolve_cert_reqs(candidate):
@@ -183,18 +184,18 @@ def resolve_cert_reqs(candidate):
     the wrap_socket function/method from the ssl module.
     Defaults to :data:`ssl.CERT_NONE`.
     If given a string it is assumed to be the name of the constant in the
-    :mod:`ssl` module or its abbrevation.
+    :mod:`ssl` module or its abbreviation.
     (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
     If it's neither `None` nor a string we assume it is already the numeric
     constant which can directly be passed to wrap_socket.
     """
     if candidate is None:
-        return CERT_NONE
+        return CERT_REQUIRED
 
     if isinstance(candidate, str):
         res = getattr(ssl, candidate, None)
         if res is None:
-            res = getattr(ssl, 'CERT_' + candidate)
+            res = getattr(ssl, "CERT_" + candidate)
         return res
 
     return candidate
@@ -205,19 +206,20 @@ def resolve_ssl_version(candidate):
     like resolve_cert_reqs
     """
     if candidate is None:
-        return PROTOCOL_SSLv23
+        return PROTOCOL_TLS
 
     if isinstance(candidate, str):
         res = getattr(ssl, candidate, None)
         if res is None:
-            res = getattr(ssl, 'PROTOCOL_' + candidate)
+            res = getattr(ssl, "PROTOCOL_" + candidate)
         return res
 
     return candidate
 
 
-def create_urllib3_context(ssl_version=None, cert_reqs=None,
-                           options=None, ciphers=None):
+def create_urllib3_context(
+    ssl_version=None, cert_reqs=None, options=None, ciphers=None
+):
     """All arguments have the same meaning as ``ssl_wrap_socket``.
 
     By default, this function does a lot of the same work that
@@ -251,7 +253,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None,
         Constructed SSLContext object with specified options
     :rtype: SSLContext
     """
-    context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
+    context = SSLContext(ssl_version or PROTOCOL_TLS)
+
+    context.set_ciphers(ciphers or DEFAULT_CIPHERS)
 
     # Setting the default here, as we may have no ssl module on import
     cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
@@ -268,21 +272,40 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None,
 
     context.options |= options
 
-    if getattr(context, 'supports_set_ciphers', True):  # Platform-specific: Python 2.6
-        context.set_ciphers(ciphers or DEFAULT_CIPHERS)
+    # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
+    # necessary for conditional client cert authentication with TLS 1.3.
+    # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older
+    # versions of Python.  We only enable on Python 3.7.4+ or if certificate
+    # verification is enabled to work around Python issue #37428
+    # See: https://bugs.python.org/issue37428
+    if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr(
+        context, "post_handshake_auth", None
+    ) is not None:
+        context.post_handshake_auth = True
 
     context.verify_mode = cert_reqs
-    if getattr(context, 'check_hostname', None) is not None:  # Platform-specific: Python 3.2
+    if (
+        getattr(context, "check_hostname", None) is not None
+    ):  # Platform-specific: Python 3.2
         # We do our own verification, including fingerprints and alternative
         # hostnames. So disable it here
         context.check_hostname = False
     return context
 
 
-def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
-                    ca_certs=None, server_hostname=None,
-                    ssl_version=None, ciphers=None, ssl_context=None,
-                    ca_cert_dir=None):
+def ssl_wrap_socket(
+    sock,
+    keyfile=None,
+    certfile=None,
+    cert_reqs=None,
+    ca_certs=None,
+    server_hostname=None,
+    ssl_version=None,
+    ciphers=None,
+    ssl_context=None,
+    ca_cert_dir=None,
+    key_password=None,
+):
     """
     All arguments except for server_hostname, ssl_context, and ca_cert_dir have
     the same meaning as they do when using :func:`ssl.wrap_socket`.
@@ -293,25 +316,25 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
         A pre-made :class:`SSLContext` object. If none is provided, one will
         be created using :func:`create_urllib3_context`.
     :param ciphers:
-        A string of ciphers we wish the client to support. This is not
-        supported on Python 2.6 as the ssl module does not support it.
+        A string of ciphers we wish the client to support.
     :param ca_cert_dir:
         A directory containing CA certificates in multiple separate files, as
         supported by OpenSSL's -CApath flag or the capath argument to
         SSLContext.load_verify_locations().
+    :param key_password:
+        Optional password if the keyfile is encrypted.
     """
     context = ssl_context
     if context is None:
         # Note: This branch of code and all the variables in it are no longer
         # used by urllib3 itself. We should consider deprecating and removing
         # this code.
-        context = create_urllib3_context(ssl_version, cert_reqs,
-                                         ciphers=ciphers)
+        context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
 
     if ca_certs or ca_cert_dir:
         try:
             context.load_verify_locations(ca_certs, ca_cert_dir)
-        except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
+        except IOError as e:  # Platform-specific: Python 2.7
             raise SSLError(e)
         # Py33 raises FileNotFoundError which subclasses OSError
         # These are not equivalent unless we check the errno attribute
@@ -319,23 +342,66 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
             if e.errno == errno.ENOENT:
                 raise SSLError(e)
             raise
-    elif getattr(context, 'load_default_certs', None) is not None:
+
+    elif ssl_context is None and hasattr(context, "load_default_certs"):
         # try to load OS default certs; works well on Windows (require Python3.4+)
         context.load_default_certs()
 
-    if certfile:
-        context.load_cert_chain(certfile, keyfile)
-    if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
-        return context.wrap_socket(sock, server_hostname=server_hostname)
+    # Attempt to detect if we get the goofy behavior of the
+    # keyfile being encrypted and OpenSSL asking for the
+    # passphrase via the terminal and instead error out.
+    if keyfile and key_password is None and _is_key_file_encrypted(keyfile):
+        raise SSLError("Client private key is encrypted, password is required")
+
+    if certfile:
+        if key_password is None:
+            context.load_cert_chain(certfile, keyfile)
+        else:
+            context.load_cert_chain(certfile, keyfile, key_password)
+
+    # If we detect server_hostname is an IP address then the SNI
+    # extension should not be used according to RFC3546 Section 3.1
+    # We shouldn't warn the user if SNI isn't available but we would
+    # not be using SNI anyways due to IP address for server_hostname.
+    if (
+        server_hostname is not None and not is_ipaddress(server_hostname)
+    ) or IS_SECURETRANSPORT:
+        if HAS_SNI and server_hostname is not None:
+            return context.wrap_socket(sock, server_hostname=server_hostname)
+
+        warnings.warn(
+            "An HTTPS request has been made, but the SNI (Server Name "
+            "Indication) extension to TLS is not available on this platform. "
+            "This may cause the server to present an incorrect TLS "
+            "certificate, which can cause validation failures. You can upgrade to "
+            "a newer version of Python to solve this. For more information, see "
+            "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
+            "#ssl-warnings",
+            SNIMissingWarning,
+        )
 
-    warnings.warn(
-        'An HTTPS request has been made, but the SNI (Subject Name '
-        'Indication) extension to TLS is not available on this platform. '
-        'This may cause the server to present an incorrect TLS '
-        'certificate, which can cause validation failures. You can upgrade to '
-        'a newer version of Python to solve this. For more information, see '
-        'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
-        '#ssl-warnings',
-        SNIMissingWarning
-    )
     return context.wrap_socket(sock)
+
+
+def is_ipaddress(hostname):
+    """Detects whether the hostname given is an IPv4 or IPv6 address.
+    Also detects IPv6 addresses with Zone IDs.
+
+    :param str hostname: Hostname to examine.
+    :return: True if the hostname is an IP address, False otherwise.
+    """
+    if not six.PY2 and isinstance(hostname, bytes):
+        # IDN A-label bytes are ASCII compatible.
+        hostname = hostname.decode("ascii")
+    return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname))
+
+
+def _is_key_file_encrypted(key_file):
+    """Detects if a key file is encrypted or not."""
+    with open(key_file, "r") as f:
+        for line in f:
+            # Look for Proc-Type: 4,ENCRYPTED
+            if "ENCRYPTED" in line:
+                return True
+
+    return False
diff --git a/lib/urllib3/util/timeout.py b/lib/urllib3/util/timeout.py
index cec817e6..c1dc1e97 100644
--- a/lib/urllib3/util/timeout.py
+++ b/lib/urllib3/util/timeout.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 # The default socket timeout, used by httplib to indicate that no timeout was
 # specified by the user
 from socket import _GLOBAL_DEFAULT_TIMEOUT
@@ -45,19 +46,20 @@ class Timeout(object):
     :type total: integer, float, or None
 
     :param connect:
-        The maximum amount of time to wait for a connection attempt to a server
-        to succeed. Omitting the parameter will default the connect timeout to
-        the system default, probably `the global default timeout in socket.py
+        The maximum amount of time (in seconds) to wait for a connection
+        attempt to a server to succeed. Omitting the parameter will default the
+        connect timeout to the system default, probably `the global default
+        timeout in socket.py
         `_.
         None will set an infinite timeout for connection attempts.
 
     :type connect: integer, float, or None
 
     :param read:
-        The maximum amount of time to wait between consecutive
-        read operations for a response from the server. Omitting
-        the parameter will default the read timeout to the system
-        default, probably `the global default timeout in socket.py
+        The maximum amount of time (in seconds) to wait between consecutive
+        read operations for a response from the server. Omitting the parameter
+        will default the read timeout to the system default, probably `the
+        global default timeout in socket.py
         `_.
         None will set an infinite timeout.
 
@@ -91,14 +93,18 @@ class Timeout(object):
     DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT
 
     def __init__(self, total=None, connect=_Default, read=_Default):
-        self._connect = self._validate_timeout(connect, 'connect')
-        self._read = self._validate_timeout(read, 'read')
-        self.total = self._validate_timeout(total, 'total')
+        self._connect = self._validate_timeout(connect, "connect")
+        self._read = self._validate_timeout(read, "read")
+        self.total = self._validate_timeout(total, "total")
         self._start_connect = None
 
     def __str__(self):
-        return '%s(connect=%r, read=%r, total=%r)' % (
-            type(self).__name__, self._connect, self._read, self.total)
+        return "%s(connect=%r, read=%r, total=%r)" % (
+            type(self).__name__,
+            self._connect,
+            self._read,
+            self.total,
+        )
 
     @classmethod
     def _validate_timeout(cls, value, name):
@@ -118,22 +124,31 @@ class Timeout(object):
             return value
 
         if isinstance(value, bool):
-            raise ValueError("Timeout cannot be a boolean value. It must "
-                             "be an int, float or None.")
+            raise ValueError(
+                "Timeout cannot be a boolean value. It must "
+                "be an int, float or None."
+            )
         try:
             float(value)
         except (TypeError, ValueError):
-            raise ValueError("Timeout value %s was %s, but it must be an "
-                             "int, float or None." % (name, value))
+            raise ValueError(
+                "Timeout value %s was %s, but it must be an "
+                "int, float or None." % (name, value)
+            )
 
         try:
             if value <= 0:
-                raise ValueError("Attempted to set %s timeout to %s, but the "
-                                 "timeout cannot be set to a value less "
-                                 "than or equal to 0." % (name, value))
-        except TypeError:  # Python 3
-            raise ValueError("Timeout value %s was %s, but it must be an "
-                             "int, float or None." % (name, value))
+                raise ValueError(
+                    "Attempted to set %s timeout to %s, but the "
+                    "timeout cannot be set to a value less "
+                    "than or equal to 0." % (name, value)
+                )
+        except TypeError:
+            # Python 3
+            raise ValueError(
+                "Timeout value %s was %s, but it must be an "
+                "int, float or None." % (name, value)
+            )
 
         return value
 
@@ -165,8 +180,7 @@ class Timeout(object):
         # We can't use copy.deepcopy because that will also create a new object
         # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
         # detect the user default.
-        return Timeout(connect=self._connect, read=self._read,
-                       total=self.total)
+        return Timeout(connect=self._connect, read=self._read, total=self.total)
 
     def start_connect(self):
         """ Start the timeout clock, used during a connect() attempt
@@ -182,14 +196,15 @@ class Timeout(object):
     def get_connect_duration(self):
         """ Gets the time elapsed since the call to :meth:`start_connect`.
 
-        :return: Elapsed time.
+        :return: Elapsed time in seconds.
         :rtype: float
         :raises urllib3.exceptions.TimeoutStateError: if you attempt
             to get duration for a timer that hasn't been started.
         """
         if self._start_connect is None:
-            raise TimeoutStateError("Can't get connect duration for timer "
-                                    "that has not started.")
+            raise TimeoutStateError(
+                "Can't get connect duration for timer " "that has not started."
+            )
         return current_time() - self._start_connect
 
     @property
@@ -227,15 +242,16 @@ class Timeout(object):
         :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
             has not yet been called on this object.
         """
-        if (self.total is not None and
-                self.total is not self.DEFAULT_TIMEOUT and
-                self._read is not None and
-                self._read is not self.DEFAULT_TIMEOUT):
+        if (
+            self.total is not None
+            and self.total is not self.DEFAULT_TIMEOUT
+            and self._read is not None
+            and self._read is not self.DEFAULT_TIMEOUT
+        ):
             # In case the connect timeout has not yet been established.
             if self._start_connect is None:
                 return self._read
-            return max(0, min(self.total - self.get_connect_duration(),
-                              self._read))
+            return max(0, min(self.total - self.get_connect_duration(), self._read))
         elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT:
             return max(0, self.total - self.get_connect_duration())
         else:
diff --git a/lib/urllib3/util/url.py b/lib/urllib3/util/url.py
index 6b6f9968..9675f742 100644
--- a/lib/urllib3/util/url.py
+++ b/lib/urllib3/util/url.py
@@ -1,34 +1,110 @@
 from __future__ import absolute_import
+import re
 from collections import namedtuple
 
 from ..exceptions import LocationParseError
+from ..packages import six
 
 
-url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']
+url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"]
 
 # We only want to normalize urls with an HTTP(S) scheme.
 # urllib3 infers URLs without a scheme (None) to be http.
-NORMALIZABLE_SCHEMES = ('http', 'https', None)
+NORMALIZABLE_SCHEMES = ("http", "https", None)
+
+# Almost all of these patterns were derived from the
+# 'rfc3986' module: https://github.com/python-hyper/rfc3986
+PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")
+SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")
+URI_RE = re.compile(
+    r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"
+    r"(?://([^/?#]*))?"
+    r"([^?#]*)"
+    r"(?:\?([^#]*))?"
+    r"(?:#(.*))?$",
+    re.UNICODE | re.DOTALL,
+)
+
+IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"
+HEX_PAT = "[0-9A-Fa-f]{1,4}"
+LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT)
+_subs = {"hex": HEX_PAT, "ls32": LS32_PAT}
+_variations = [
+    #                            6( h16 ":" ) ls32
+    "(?:%(hex)s:){6}%(ls32)s",
+    #                       "::" 5( h16 ":" ) ls32
+    "::(?:%(hex)s:){5}%(ls32)s",
+    # [               h16 ] "::" 4( h16 ":" ) ls32
+    "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",
+    # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
+    "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",
+    # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
+    "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",
+    # [ *3( h16 ":" ) h16 ] "::"    h16 ":"   ls32
+    "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",
+    # [ *4( h16 ":" ) h16 ] "::"              ls32
+    "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",
+    # [ *5( h16 ":" ) h16 ] "::"              h16
+    "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",
+    # [ *6( h16 ":" ) h16 ] "::"
+    "(?:(?:%(hex)s:){0,6}%(hex)s)?::",
+]
+
+UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._!\-~"
+IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"
+ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"
+IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]"
+REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"
+TARGET_RE = re.compile(r"^(/[^?]*)(?:\?([^#]+))?(?:#(.*))?$")
+
+IPV4_RE = re.compile("^" + IPV4_PAT + "$")
+IPV6_RE = re.compile("^" + IPV6_PAT + "$")
+IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$")
+BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$")
+ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$")
+
+SUBAUTHORITY_PAT = (u"^(?:(.*)@)?(%s|%s|%s)(?::([0-9]{0,5}))?$") % (
+    REG_NAME_PAT,
+    IPV4_PAT,
+    IPV6_ADDRZ_PAT,
+)
+SUBAUTHORITY_RE = re.compile(SUBAUTHORITY_PAT, re.UNICODE | re.DOTALL)
+
+UNRESERVED_CHARS = set(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"
+)
+SUB_DELIM_CHARS = set("!$&'()*+,;=")
+USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"}
+PATH_CHARS = USERINFO_CHARS | {"@", "/"}
+QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"}
 
 
-class Url(namedtuple('Url', url_attrs)):
+class Url(namedtuple("Url", url_attrs)):
     """
-    Datastructure for representing an HTTP URL. Used as a return value for
+    Data structure for representing an HTTP URL. Used as a return value for
     :func:`parse_url`. Both the scheme and host are normalized as they are
     both case-insensitive according to RFC 3986.
     """
+
     __slots__ = ()
 
-    def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
-                query=None, fragment=None):
-        if path and not path.startswith('/'):
-            path = '/' + path
-        if scheme:
+    def __new__(
+        cls,
+        scheme=None,
+        auth=None,
+        host=None,
+        port=None,
+        path=None,
+        query=None,
+        fragment=None,
+    ):
+        if path and not path.startswith("/"):
+            path = "/" + path
+        if scheme is not None:
             scheme = scheme.lower()
-        if host and scheme in NORMALIZABLE_SCHEMES:
-            host = host.lower()
-        return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
-                                       query, fragment)
+        return super(Url, cls).__new__(
+            cls, scheme, auth, host, port, path, query, fragment
+        )
 
     @property
     def hostname(self):
@@ -38,10 +114,10 @@ class Url(namedtuple('Url', url_attrs)):
     @property
     def request_uri(self):
         """Absolute path including the query string."""
-        uri = self.path or '/'
+        uri = self.path or "/"
 
         if self.query is not None:
-            uri += '?' + self.query
+            uri += "?" + self.query
 
         return uri
 
@@ -49,7 +125,7 @@ class Url(namedtuple('Url', url_attrs)):
     def netloc(self):
         """Network location including host and port"""
         if self.port:
-            return '%s:%d' % (self.host, self.port)
+            return "%s:%d" % (self.host, self.port)
         return self.host
 
     @property
@@ -72,23 +148,23 @@ class Url(namedtuple('Url', url_attrs)):
             'http://username:password@host.com:80/path?query#fragment'
         """
         scheme, auth, host, port, path, query, fragment = self
-        url = ''
+        url = u""
 
         # We use "is not None" we want things to happen with empty strings (or 0 port)
         if scheme is not None:
-            url += scheme + '://'
+            url += scheme + u"://"
         if auth is not None:
-            url += auth + '@'
+            url += auth + u"@"
         if host is not None:
             url += host
         if port is not None:
-            url += ':' + str(port)
+            url += u":" + str(port)
         if path is not None:
             url += path
         if query is not None:
-            url += '?' + query
+            url += u"?" + query
         if fragment is not None:
-            url += '#' + fragment
+            url += u"#" + fragment
 
         return url
 
@@ -98,6 +174,8 @@ class Url(namedtuple('Url', url_attrs)):
 
 def split_first(s, delims):
     """
+    .. deprecated:: 1.25
+
     Given a string and an iterable of delimiters, split on the first found
     delimiter. Return two split parts and the matched delimiter.
 
@@ -124,15 +202,150 @@ def split_first(s, delims):
             min_delim = d
 
     if min_idx is None or min_idx < 0:
-        return s, '', None
+        return s, "", None
 
-    return s[:min_idx], s[min_idx + 1:], min_delim
+    return s[:min_idx], s[min_idx + 1 :], min_delim
+
+
+def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"):
+    """Percent-encodes a URI component without reapplying
+    onto an already percent-encoded component.
+    """
+    if component is None:
+        return component
+
+    component = six.ensure_text(component)
+
+    # Try to see if the component we're encoding is already percent-encoded
+    # so we can skip all '%' characters but still encode all others.
+    percent_encodings = PERCENT_RE.findall(component)
+
+    # Normalize existing percent-encoded bytes.
+    for enc in percent_encodings:
+        if not enc.isupper():
+            component = component.replace(enc, enc.upper())
+
+    uri_bytes = component.encode("utf-8", "surrogatepass")
+    is_percent_encoded = len(percent_encodings) == uri_bytes.count(b"%")
+
+    encoded_component = bytearray()
+
+    for i in range(0, len(uri_bytes)):
+        # Will return a single character bytestring on both Python 2 & 3
+        byte = uri_bytes[i : i + 1]
+        byte_ord = ord(byte)
+        if (is_percent_encoded and byte == b"%") or (
+            byte_ord < 128 and byte.decode() in allowed_chars
+        ):
+            encoded_component.extend(byte)
+            continue
+        encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper()))
+
+    return encoded_component.decode(encoding)
+
+
+def _remove_path_dot_segments(path):
+    # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
+    segments = path.split("/")  # Turn the path into a list of segments
+    output = []  # Initialize the variable to use to store output
+
+    for segment in segments:
+        # '.' is the current directory, so ignore it, it is superfluous
+        if segment == ".":
+            continue
+        # Anything other than '..', should be appended to the output
+        elif segment != "..":
+            output.append(segment)
+        # In this case segment == '..', if we can, we should pop the last
+        # element
+        elif output:
+            output.pop()
+
+    # If the path starts with '/' and the output is empty or the first string
+    # is non-empty
+    if path.startswith("/") and (not output or output[0]):
+        output.insert(0, "")
+
+    # If the path starts with '/.' or '/..' ensure we add one more empty
+    # string to add a trailing '/'
+    if path.endswith(("/.", "/..")):
+        output.append("")
+
+    return "/".join(output)
+
+
+def _normalize_host(host, scheme):
+    if host:
+        if isinstance(host, six.binary_type):
+            host = six.ensure_str(host)
+
+        if scheme in NORMALIZABLE_SCHEMES:
+            is_ipv6 = IPV6_ADDRZ_RE.match(host)
+            if is_ipv6:
+                match = ZONE_ID_RE.search(host)
+                if match:
+                    start, end = match.span(1)
+                    zone_id = host[start:end]
+
+                    if zone_id.startswith("%25") and zone_id != "%25":
+                        zone_id = zone_id[3:]
+                    else:
+                        zone_id = zone_id[1:]
+                    zone_id = "%" + _encode_invalid_chars(zone_id, UNRESERVED_CHARS)
+                    return host[:start].lower() + zone_id + host[end:]
+                else:
+                    return host.lower()
+            elif not IPV4_RE.match(host):
+                return six.ensure_str(
+                    b".".join([_idna_encode(label) for label in host.split(".")])
+                )
+    return host
+
+
+def _idna_encode(name):
+    if name and any([ord(x) > 128 for x in name]):
+        try:
+            import idna
+        except ImportError:
+            six.raise_from(
+                LocationParseError("Unable to parse URL without the 'idna' module"),
+                None,
+            )
+        try:
+            return idna.encode(name.lower(), strict=True, std3_rules=True)
+        except idna.IDNAError:
+            six.raise_from(
+                LocationParseError(u"Name '%s' is not a valid IDNA label" % name), None
+            )
+    return name.lower().encode("ascii")
+
+
+def _encode_target(target):
+    """Percent-encodes a request target so that there are no invalid characters"""
+    if not target.startswith("/"):
+        return target
+
+    path, query, fragment = TARGET_RE.match(target).groups()
+    target = _encode_invalid_chars(path, PATH_CHARS)
+    query = _encode_invalid_chars(query, QUERY_CHARS)
+    fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS)
+    if query is not None:
+        target += "?" + query
+    if fragment is not None:
+        target += "#" + target
+    return target
 
 
 def parse_url(url):
     """
     Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
     performed to parse incomplete urls. Fields not provided will be None.
+    This parser is RFC 3986 compliant.
+
+    The parser logic and helper functions are based heavily on
+    work done in the ``rfc3986`` module.
+
+    :param str url: URL to parse into a :class:`.Url` namedtuple.
 
     Partly backwards-compatible with :mod:`urlparse`.
 
@@ -145,81 +358,77 @@ def parse_url(url):
         >>> parse_url('/foo?bar')
         Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
     """
-
-    # While this code has overlap with stdlib's urlparse, it is much
-    # simplified for our needs and less annoying.
-    # Additionally, this implementations does silly things to be optimal
-    # on CPython.
-
     if not url:
         # Empty
         return Url()
 
-    scheme = None
-    auth = None
-    host = None
-    port = None
-    path = None
-    fragment = None
-    query = None
+    source_url = url
+    if not SCHEME_RE.search(url):
+        url = "//" + url
 
-    # Scheme
-    if '://' in url:
-        scheme, url = url.split('://', 1)
+    try:
+        scheme, authority, path, query, fragment = URI_RE.match(url).groups()
+        normalize_uri = scheme is None or scheme.lower() in NORMALIZABLE_SCHEMES
 
-    # Find the earliest Authority Terminator
-    # (http://tools.ietf.org/html/rfc3986#section-3.2)
-    url, path_, delim = split_first(url, ['/', '?', '#'])
+        if scheme:
+            scheme = scheme.lower()
 
-    if delim:
-        # Reassemble the path
-        path = delim + path_
-
-    # Auth
-    if '@' in url:
-        # Last '@' denotes end of auth part
-        auth, url = url.rsplit('@', 1)
-
-    # IPv6
-    if url and url[0] == '[':
-        host, url = url.split(']', 1)
-        host += ']'
-
-    # Port
-    if ':' in url:
-        _host, port = url.split(':', 1)
-
-        if not host:
-            host = _host
-
-        if port:
-            # If given, ports must be integers. No whitespace, no plus or
-            # minus prefixes, no non-integer digits such as ^2 (superscript).
-            if not port.isdigit():
-                raise LocationParseError(url)
-            try:
-                port = int(port)
-            except ValueError:
-                raise LocationParseError(url)
+        if authority:
+            auth, host, port = SUBAUTHORITY_RE.match(authority).groups()
+            if auth and normalize_uri:
+                auth = _encode_invalid_chars(auth, USERINFO_CHARS)
+            if port == "":
+                port = None
         else:
-            # Blank ports are cool, too. (rfc3986#section-3.2.3)
-            port = None
+            auth, host, port = None, None, None
 
-    elif not host and url:
-        host = url
+        if port is not None:
+            port = int(port)
+            if not (0 <= port <= 65535):
+                raise LocationParseError(url)
 
+        host = _normalize_host(host, scheme)
+
+        if normalize_uri and path:
+            path = _remove_path_dot_segments(path)
+            path = _encode_invalid_chars(path, PATH_CHARS)
+        if normalize_uri and query:
+            query = _encode_invalid_chars(query, QUERY_CHARS)
+        if normalize_uri and fragment:
+            fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS)
+
+    except (ValueError, AttributeError):
+        return six.raise_from(LocationParseError(source_url), None)
+
+    # For the sake of backwards compatibility we put empty
+    # string values for path if there are any defined values
+    # beyond the path in the URL.
+    # TODO: Remove this when we break backwards compatibility.
     if not path:
-        return Url(scheme, auth, host, port, path, query, fragment)
+        if query is not None or fragment is not None:
+            path = ""
+        else:
+            path = None
 
-    # Fragment
-    if '#' in path:
-        path, fragment = path.split('#', 1)
+    # Ensure that each part of the URL is a `str` for
+    # backwards compatibility.
+    if isinstance(url, six.text_type):
+        ensure_func = six.ensure_text
+    else:
+        ensure_func = six.ensure_str
 
-    # Query
-    if '?' in path:
-        path, query = path.split('?', 1)
+    def ensure_type(x):
+        return x if x is None else ensure_func(x)
 
-    return Url(scheme, auth, host, port, path, query, fragment)
+    return Url(
+        scheme=ensure_type(scheme),
+        auth=ensure_type(auth),
+        host=ensure_type(host),
+        port=port,
+        path=ensure_type(path),
+        query=ensure_type(query),
+        fragment=ensure_type(fragment),
+    )
 
 
 def get_host(url):
@@ -227,4 +436,4 @@ def get_host(url):
     Deprecated. Use :func:`parse_url` instead.
     """
     p = parse_url(url)
-    return p.scheme or 'http', p.hostname, p.port
+    return p.scheme or "http", p.hostname, p.port
diff --git a/lib/urllib3/util/wait.py b/lib/urllib3/util/wait.py
index cb396e50..d71d2fd7 100644
--- a/lib/urllib3/util/wait.py
+++ b/lib/urllib3/util/wait.py
@@ -1,40 +1,153 @@
-from .selectors import (
-    HAS_SELECT,
-    DefaultSelector,
-    EVENT_READ,
-    EVENT_WRITE
-)
+import errno
+from functools import partial
+import select
+import sys
+
+try:
+    from time import monotonic
+except ImportError:
+    from time import time as monotonic
+
+__all__ = ["NoWayToWaitForSocketError", "wait_for_read", "wait_for_write"]
 
 
-def _wait_for_io_events(socks, events, timeout=None):
-    """ Waits for IO events to be available from a list of sockets
-    or optionally a single socket if passed in. Returns a list of
-    sockets that can be interacted with immediately. """
-    if not HAS_SELECT:
-        raise ValueError('Platform does not have a selector')
-    if not isinstance(socks, list):
-        # Probably just a single socket.
-        if hasattr(socks, "fileno"):
-            socks = [socks]
-        # Otherwise it might be a non-list iterable.
+class NoWayToWaitForSocketError(Exception):
+    pass
+
+
+# How should we wait on sockets?
+#
+# There are two types of APIs you can use for waiting on sockets: the fancy
+# modern stateful APIs like epoll/kqueue, and the older stateless APIs like
+# select/poll. The stateful APIs are more efficient when you have a lots of
+# sockets to keep track of, because you can set them up once and then use them
+# lots of times. But we only ever want to wait on a single socket at a time
+# and don't want to keep track of state, so the stateless APIs are actually
+# more efficient. So we want to use select() or poll().
+#
+# Now, how do we choose between select() and poll()? On traditional Unixes,
+# select() has a strange calling convention that makes it slow, or fail
+# altogether, for high-numbered file descriptors. The point of poll() is to fix
+# that, so on Unixes, we prefer poll().
+#
+# On Windows, there is no poll() (or at least Python doesn't provide a wrapper
+# for it), but that's OK, because on Windows, select() doesn't have this
+# strange calling convention; plain select() works fine.
+#
+# So: on Windows we use select(), and everywhere else we use poll(). We also
+# fall back to select() in case poll() is somehow broken or missing.
+
+if sys.version_info >= (3, 5):
+    # Modern Python, that retries syscalls by default
+    def _retry_on_intr(fn, timeout):
+        return fn(timeout)
+
+
+else:
+    # Old and broken Pythons.
+    def _retry_on_intr(fn, timeout):
+        if timeout is None:
+            deadline = float("inf")
         else:
-            socks = list(socks)
-    with DefaultSelector() as selector:
-        for sock in socks:
-            selector.register(sock, events)
-        return [key[0].fileobj for key in
-                selector.select(timeout) if key[1] & events]
+            deadline = monotonic() + timeout
+
+        while True:
+            try:
+                return fn(timeout)
+            # OSError for 3 <= pyver < 3.5, select.error for pyver <= 2.7
+            except (OSError, select.error) as e:
+                # 'e.args[0]' incantation works for both OSError and select.error
+                if e.args[0] != errno.EINTR:
+                    raise
+                else:
+                    timeout = deadline - monotonic()
+                    if timeout < 0:
+                        timeout = 0
+                    if timeout == float("inf"):
+                        timeout = None
+                    continue
 
 
-def wait_for_read(socks, timeout=None):
-    """ Waits for reading to be available from a list of sockets
-    or optionally a single socket if passed in. Returns a list of
-    sockets that can be read from immediately. """
-    return _wait_for_io_events(socks, EVENT_READ, timeout)
+def select_wait_for_socket(sock, read=False, write=False, timeout=None):
+    if not read and not write:
+        raise RuntimeError("must specify at least one of read=True, write=True")
+    rcheck = []
+    wcheck = []
+    if read:
+        rcheck.append(sock)
+    if write:
+        wcheck.append(sock)
+    # When doing a non-blocking connect, most systems signal success by
+    # marking the socket writable. Windows, though, signals success by marked
+    # it as "exceptional". We paper over the difference by checking the write
+    # sockets for both conditions. (The stdlib selectors module does the same
+    # thing.)
+    fn = partial(select.select, rcheck, wcheck, wcheck)
+    rready, wready, xready = _retry_on_intr(fn, timeout)
+    return bool(rready or wready or xready)
 
 
-def wait_for_write(socks, timeout=None):
-    """ Waits for writing to be available from a list of sockets
-    or optionally a single socket if passed in. Returns a list of
-    sockets that can be written to immediately. """
-    return _wait_for_io_events(socks, EVENT_WRITE, timeout)
+def poll_wait_for_socket(sock, read=False, write=False, timeout=None):
+    if not read and not write:
+        raise RuntimeError("must specify at least one of read=True, write=True")
+    mask = 0
+    if read:
+        mask |= select.POLLIN
+    if write:
+        mask |= select.POLLOUT
+    poll_obj = select.poll()
+    poll_obj.register(sock, mask)
+
+    # For some reason, poll() takes timeout in milliseconds
+    def do_poll(t):
+        if t is not None:
+            t *= 1000
+        return poll_obj.poll(t)
+
+    return bool(_retry_on_intr(do_poll, timeout))
+
+
+def null_wait_for_socket(*args, **kwargs):
+    raise NoWayToWaitForSocketError("no select-equivalent available")
+
+
+def _have_working_poll():
+    # Apparently some systems have a select.poll that fails as soon as you try
+    # to use it, either due to strange configuration or broken monkeypatching
+    # from libraries like eventlet/greenlet.
+    try:
+        poll_obj = select.poll()
+        _retry_on_intr(poll_obj.poll, 0)
+    except (AttributeError, OSError):
+        return False
+    else:
+        return True
+
+
+def wait_for_socket(*args, **kwargs):
+    # We delay choosing which implementation to use until the first time we're
+    # called. We could do it at import time, but then we might make the wrong
+    # decision if someone goes wild with monkeypatching select.poll after
+    # we're imported.
+    global wait_for_socket
+    if _have_working_poll():
+        wait_for_socket = poll_wait_for_socket
+    elif hasattr(select, "select"):
+        wait_for_socket = select_wait_for_socket
+    else:  # Platform-specific: Appengine.
+        wait_for_socket = null_wait_for_socket
+    return wait_for_socket(*args, **kwargs)
+
+
+def wait_for_read(sock, timeout=None):
+    """ Waits for reading to be available on a given socket.
+    Returns True if the socket is readable, or False if the timeout expired.
+    """
+    return wait_for_socket(sock, read=True, timeout=timeout)
+
+
+def wait_for_write(sock, timeout=None):
+    """ Waits for writing to be available on a given socket.
+    Returns True if the socket is readable, or False if the timeout expired.
+    """
+    return wait_for_socket(sock, write=True, timeout=timeout)
diff --git a/specials/search.py b/specials/search.py
index b7d14d4a..bf5fc082 100755
--- a/specials/search.py
+++ b/specials/search.py
@@ -1,157 +1,347 @@
 # -*- coding: utf-8 -*-
+# -*- Channel New Search -*-
+# -*- Created for Alfa-addon -*-
+# -*- By the Alfa Develop Group -*-
 
-import glob
 import os
-import re
+import json
 import time
-from threading import Thread
-
-import xbmcaddon
-
-from channelselector import get_thumb, auto_filter
-from core import channeltools
-from core import scrapertools
-from core import tmdb
+from lib.concurrent import futures
 from core.item import Item
-from platformcode import config, logger
+from core import tmdb
+from core import scrapertools
+from core import channeltools
+import channelselector
+from channelselector import get_thumb
+from platformcode import logger
+from platformcode import config
 from platformcode import platformtools
-from core.support import typo
-
-addon = xbmcaddon.Addon('metadata.themoviedb.org')
-def_lang = addon.getSetting('language')
-
-link_list = []
-max_links = 30
+from platformcode import unify
 
 
 def mainlist(item):
     logger.info()
-    item.channel = "search"
 
-    itemlist = []
-    context = [{"title": config.get_localized_string(60412), "action": "setting_channel", "channel": item.channel}]
-    itemlist.append(Item(channel=item.channel, action="sub_menu", title="[B]" + config.get_localized_string(70305)+ "[/B]", context=context,
+    itemlist = list()
+
+    itemlist.append(Item(channel=item.channel, title='Búsqueda General', action='new_search', mode='all',
+                         thumbnail=get_thumb("search_generic.png")))
+
+    itemlist.append(Item(channel=item.channel, title='Buscar Película', action='new_search', mode='movie',
                          thumbnail=get_thumb("search.png")))
-    itemlist.append(Item(channel=item.channel, action='genres_menu', title=config.get_localized_string(70306), type='movie',
-                         thumbnail=get_thumb("genres.png")))
-    itemlist.append (Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70307),
-                          context=context, search_type='list', list_type='movie/popular',
-                          thumbnail=get_thumb("popular.png")))
-    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70308),
-                         context=context, search_type='list', list_type='movie/top_rated',
-                         thumbnail=get_thumb("top_rated.png")))
-    #itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70309), context=context,
-    #         search_type='list', list_type='movie/now_playing',
-    #                     thumbnail=get_thumb("now_playing.png")))
-    itemlist.append(Item(channel=item.channel, action='genres_menu', title=config.get_localized_string(70310), type='tv',
-                         thumbnail=get_thumb("genres.png")))
-    itemlist.append(
-        Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70311), context=context,
-             search_type='list',list_type='tv/popular', thumbnail=get_thumb("popular.png")))
-    #itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70312), context=context,
-    #                     search_type='list', list_type='tv/on_the_air', thumbnail=get_thumb("on_the_air.png")))
-    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70313), context=context,
-                         search_type='list', list_type='tv/top_rated', thumbnail=get_thumb("top_rated.png")))
 
+    itemlist.append(Item(channel=item.channel, title='Buscar Serie', action='new_search', mode='tvshow',
+                         thumbnail=get_thumb("search_tvshow.png")))
+
+    itemlist.append(Item(channel=item.channel, title='Buscar Actor/Actriz', action='new_search',
+                         page=1, mode='person', thumbnail=get_thumb("search_star.png")))
+
+    itemlist.append(Item(channel=item.channel, title='Más opciones de búsqueda', action='sub_menu',
+                         thumbnail=get_thumb('search_more.png')))
+
+    itemlist.append(Item(channel=item.channel, title='Seleccionar canales para la búsqueda', action='opciones',
+                         thumbnail=get_thumb('setting_0.png')))
+
+    itemlist = set_context(itemlist)
 
     return itemlist
 
 
-def genres_menu(item):
-    from channelselector import thumb
-
-    itemlist = []
-
-    genres = tmdb.get_genres(item.type)
-
-    logger.debug(genres)
-    logger.debug(genres[item.type])
-
-    for key, value in genres[item.type].items():
-        itemlist.append(item.clone(title=value, action='discover_list', search_type='discover',
-                                   list_type=key, page='1'))
-    return thumb(sorted(itemlist, key=lambda it: it.title), genre=True)
-
 def sub_menu(item):
     logger.info()
-    item.channel = "search"
 
     itemlist = list()
-    context = [{"title": config.get_localized_string(70273),
-                "action": "setting_channel",
-                "channel": item.channel}]
-    itemlist.append(Item(channel=item.channel, action="search",
-                         title=config.get_localized_string(30980), context=context,
-                         thumbnail=get_thumb("search.png")))
-
-    thumbnail = get_thumb("search_star.png")
-
-    itemlist.append(Item(channel='tvmoviedb', title=config.get_localized_string(70036), action="search_",
-                         search={'url': 'search/person', 'language': def_lang, 'page': 1}, star=True,
-                         thumbnail=thumbnail))
-
-    itemlist.append(Item(channel=item.channel, action="search",
-                         title=config.get_localized_string(59998), extra="categorias",
-                         context=context,
-                         thumbnail=get_thumb("search.png")))
-    itemlist.append(Item(channel=item.channel, action="opciones", title=config.get_localized_string(59997),
-                         thumbnail=get_thumb("search.png")))
 
     itemlist.append(Item(channel="tvmoviedb", action="mainlist", title=config.get_localized_string(70274),
                          thumbnail=get_thumb("search.png")))
 
-    saved_searches_list = get_saved_searches()
-    context2 = context[:]
-    context2.append({"title": config.get_localized_string(59996),
-                     "action": "clear_saved_searches",
-                     "channel": item.channel})
-    logger.info("saved_searches_list=%s" % saved_searches_list)
+    itemlist.append(Item(channel=item.channel, action='genres_menu', title=config.get_localized_string(70306),
+                         mode='movie', thumbnail=get_thumb("genres.png")))
 
-    if saved_searches_list:
-        itemlist.append(Item(channel=item.channel, action="",
-                             title=config.get_localized_string(59995), context=context2,
-                             thumbnail=get_thumb("search.png")))
-        for saved_search_text in saved_searches_list:
-            itemlist.append(Item(channel=item.channel, action="do_search",
-                                 title='    "' + saved_search_text + '"',
-                                 extra=saved_search_text, context=context2,
-                                 category=saved_search_text,
-                                 thumbnail=get_thumb("search.png")))
+    itemlist.append(Item(channel=item.channel, action='years_menu', title='Películas por Año',
+                         mode='movie', thumbnail=get_thumb("years.png")))
+
+    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70307),
+                         search_type='list', list_type='movie/popular', mode='movie',
+                         thumbnail=get_thumb("popular.png")))
+
+    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70308),
+                         search_type='list', list_type='movie/top_rated', mode='movie',
+                         thumbnail=get_thumb("top_rated.png")))
+
+    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70309),
+                         search_type='list', list_type='movie/now_playing', mode='movie',
+                         thumbnail=get_thumb("now_playing.png")))
+
+    itemlist.append(Item(channel=item.channel, action='genres_menu', title=config.get_localized_string(70310),
+                         mode='tvshow', thumbnail=get_thumb("genres.png")))
+
+    itemlist.append(Item(channel=item.channel, action='years_menu', title='Series por Año',
+                         mode='tvshow', thumbnail=get_thumb("years.png")))
+
+    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70311),
+                         search_type='list', list_type='tv/popular', mode='tvshow',
+                         thumbnail=get_thumb("popular.png")))
+
+    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70312),
+                         search_type='list', list_type='tv/on_the_air', mode='tvshow',
+                         thumbnail=get_thumb("on_the_air.png")))
+
+    itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70313),
+                         search_type='list', list_type='tv/top_rated', mode='tvshow',
+                         thumbnail=get_thumb("top_rated.png")))
+
+    itemlist = set_context(itemlist)
 
     return itemlist
 
 
+def new_search(item):
+    logger.info()
+
+    itemlist = []
+
+    last_search = channeltools.get_channel_setting('Last_searched', 'search', '')
+    searched_text = platformtools.dialog_input(default=last_search, heading='Alfa (Busqueda)')
+
+    if not searched_text:
+        return
+
+    channeltools.set_channel_setting('Last_searched', searched_text, 'search')
+    searched_text = searched_text.replace("+", " ")
+
+    if item.mode == 'person':
+        item.searched_text = searched_text
+        return actor_list(item)
+
+    if item.mode != 'all':
+        tmdb_info = tmdb.Tmdb(texto_buscado=searched_text, tipo=item.mode.replace('show', ''))
+        results = tmdb_info.results
+        for result in results:
+            result = tmdb_info.get_infoLabels(result, origen=result)
+            if item.mode == 'movie':
+                title = result['title']
+            else:
+                title = result['name']
+                item.mode = 'tvshow'
+
+            thumbnail = result.get('thumbnail', '')
+            fanart = result.get('fanart', '')
+
+            new_item = Item(channel=item.channel,
+                            action='channel_search',
+                            title=title,
+                            text=searched_text,
+                            thumbnail=thumbnail,
+                            fanart=fanart,
+                            mode=item.mode,
+                            infoLabels=result)
+
+            if item.mode == 'movie':
+                new_item.contentTitle = result['title']
+            else:
+                new_item.contentSerieName = result['name']
+
+            itemlist.append(new_item)
+
+    if item.mode == 'all' or not itemlist:
+        itemlist = channel_search(Item(channel=item.channel,
+                                       title=searched_text,
+                                       text=searched_text,
+                                       mode='all',
+                                       infoLabels={}))
+
+    return itemlist
+
+
+def channel_search(item):
+    logger.info()
+
+    start = time.time()
+    searching = list()
+    results = list()
+    valid = list()
+    ch_list = dict()
+    to_temp = dict()
+    mode = item.mode
+    max_results = 10
+
+    searched_id = item.infoLabels['tmdb_id']
+
+    channel_list = get_channels(item)
+
+    searching += channel_list
+    cnt = 0
+
+    progress = platformtools.dialog_progress('Buscando %s ...' % item.title, "Restan %s canales" % len(channel_list),
+                                             str(searching))
+    config.set_setting('tmdb_active', False)
+
+    with futures.ThreadPoolExecutor() as executor:
+        c_results = [executor.submit(get_channel_results, ch, item) for ch in channel_list]
+
+        for res in futures.as_completed(c_results):
+            cnt += 1
+            finished = res.result()[0]
+            if res.result()[1]:
+                ch_list[res.result()[0]] = res.result()[1]
+
+            if progress.iscanceled():
+                break
+            if finished in searching:
+                searching.remove(finished)
+                progress.update((cnt * 100) / len(channel_list), "Restan %s canales" % str(len(channel_list) - cnt),
+                                str(searching))
+
+    progress.close()
+
+    cnt = 0
+    progress = platformtools.dialog_progress('Buscando %s ...' % item.title, "Preparando Resultados",
+                                             'Por favor espere...')
+
+    config.set_setting('tmdb_active', True)
+
+    for key, value in ch_list.items():
+        grouped = list()
+        cnt += 1
+        progress.update((cnt * 100) / len(ch_list), "Preparando Resultados", 'Por favor espere...')
+        if len(value) <= max_results and item.mode != 'all':
+            if len(value) == 1:
+                if not value[0].action or 'siguiente' in value[0].title.lower():
+                    continue
+            tmdb.set_infoLabels_itemlist(value, True, forced=True)
+            for elem in value:
+                if not elem.infoLabels.get('year', ""):
+                    elem.infoLabels['year'] = '-'
+                    tmdb.set_infoLabels_item(elem, True)
+
+                if elem.infoLabels['tmdb_id'] == searched_id:
+                    elem.from_channel = key
+                    if not config.get_setting('unify'):
+                        elem.title += ' [%s]' % key
+                    valid.append(elem)
+
+        for it in value:
+            if it.channel == item.channel:
+                it.channel = key
+            if it in valid:
+                continue
+            if mode == 'all' or (it.contentType and mode == it.contentType):
+                grouped.append(it)
+            elif (mode == 'movie' and it.contentTitle) or (mode == 'tvshow' and (it.contentSerieName or it.show)):
+                grouped.append(it)
+            else:
+                continue
+
+        if not grouped:
+            continue
+        to_temp[key] = grouped
+
+        if not config.get_setting('unify'):
+            title = '[COLOR yellow](%s) RESULTADOS EN[/COLOR] [%s]' % (len(grouped), key)
+        else:
+            title = '[COLOR yellow](%s) RESULTADOS EN[/COLOR]' % (len(grouped))
+
+        ch_thumb = channeltools.get_channel_parameters(key)['thumbnail']
+        results.append(Item(channel='search', title=title,
+                            action='get_from_temp', thumbnail=ch_thumb, from_channel=key, page=1))
+
+    results = sorted(results, key=lambda it: it.from_channel)
+
+    send_to_temp(to_temp)
+    config.set_setting('tmdb_active', True)
+
+    logger.debug('Finalizada en %s segs (%s canales)' % (str(time.time() - start), str(len(channel_list))))
+
+    return valid + results
+
+
+def get_channel_results(ch, item):
+    max_results = 10
+    results = list()
+
+    ch_params = channeltools.get_channel_parameters(ch)
+
+    exec "from channels import " + ch_params["channel"] + " as module"
+
+    mainlist = module.mainlist(Item(channel=ch_params["channel"]))
+    search_action = [elem for elem in mainlist if elem.action == "search"]
+
+    if search_action:
+        for search_ in search_action:
+            try:
+                results.extend(module.search(search_, item.text))
+            except:
+                pass
+    else:
+        try:
+            results.extend(module.search(item, item.text))
+        except:
+            pass
+
+    if len(results) < 0 and len(results) < max_results and item.mode != 'all':
+
+        if len(results) == 1:
+            if not results[0].action or 'siguiente' in results[0].title.lower():
+                return [ch, []]
+
+        results = get_info(results)
+
+    return [ch, results]
+
+
+def get_info(itemlist):
+    logger.info()
+    tmdb.set_infoLabels_itemlist(itemlist, True, forced=True)
+
+    return itemlist
+
+
+def get_channels(item):
+    logger.info()
+
+    channels_list = list()
+    all_channels = channelselector.filterchannels('all')
+
+    for ch in all_channels:
+        channel = ch.channel
+        ch_param = channeltools.get_channel_parameters(channel)
+        if not ch_param.get("active", False):
+            continue
+        list_cat = ch_param.get("categories", [])
+
+        if not ch_param.get("include_in_global_search", False):
+            continue
+
+        if 'anime' in list_cat:
+            n = list_cat.index('anime')
+            list_cat[n] = 'tvshow'
+
+        if item.mode == 'all' or (item.mode in list_cat):
+            if config.get_setting("include_in_global_search", channel):
+                channels_list.append(channel)
+
+    return channels_list
+
+
 def opciones(item):
-    itemlist = list()
-    itemlist.append(Item(channel=item.channel, action="setting_channel",
-                         title=config.get_localized_string(59994), folder=False,
-                         thumbnail=get_thumb("search.png")))
-    itemlist.append(Item(channel=item.channel, action="clear_saved_searches", title=config.get_localized_string(59996),
-                         folder=False, thumbnail=get_thumb("search.png")))
-    itemlist.append(Item(channel=item.channel, action="settings", title=config.get_localized_string(60531), folder=False,
-                         thumbnail=get_thumb("search.png")))
-    return itemlist
+    return setting_channel_new(item)
 
 
-def settings(item):
-    return platformtools.show_channel_settings(caption=config.get_localized_string(59993))
-
-
-def setting_channel(item):
-    if config.get_platform(True)['num_version'] >= 17.0: # A partir de Kodi 16 se puede usar multiselect, y de 17 con preselect
-        return setting_channel_new(item)
-    else:
-        return setting_channel_old(item)
-
 def setting_channel_new(item):
-    import channelselector, xbmcgui
-    from core import channeltools
+    import xbmcgui
 
     # Cargar lista de opciones (canales activos del usuario y que permitan búsqueda global)
     # ------------------------
-    lista = []; ids = []; lista_lang = []; lista_ctgs = []
+    lista = []
+    ids = []
+    lista_lang = []
+    lista_ctgs = []
     channels_list = channelselector.filterchannels('all')
     for channel in channels_list:
+        if channel.action == '':
+            continue
+
         channel_parameters = channeltools.get_channel_parameters(channel.channel)
 
         # No incluir si en la configuracion del canal no existe "include_in_global_search"
@@ -162,7 +352,7 @@ def setting_channel_new(item):
         lbl += ' %s' % ', '.join(config.get_localized_category(categ) for categ in channel_parameters['categories'])
 
         it = xbmcgui.ListItem(channel.title, lbl)
-        it.setArt({ 'thumb': channel.thumbnail, 'fanart': channel.fanart })
+        it.setArt({'thumb': channel.thumbnail, 'fanart': channel.fanart})
         lista.append(it)
         ids.append(channel.channel)
         lista_lang.append(channel_parameters['language'])
@@ -171,38 +361,39 @@ def setting_channel_new(item):
     # Diálogo para pre-seleccionar
     # ----------------------------
     preselecciones = [
-        config.get_localized_string(70570), 
-        config.get_localized_string(70571), 
-        config.get_localized_string(70572), 
-        config.get_localized_string(70573), 
-        # config.get_localized_string(70574), 
-        # config.get_localized_string(70575),
-        config.get_localized_string(70576)
+        'Buscar con la selección actual',
+        'Modificar selección actual',
+        'Modificar partiendo de Recomendados',
+        'Modificar partiendo de Frecuentes',
+        'Modificar partiendo de Todos',
+        'Modificar partiendo de Ninguno',
+        'Modificar partiendo de Castellano',
+        'Modificar partiendo de Latino'
     ]
-    presel_values = ['skip', 'actual', 'all', 'none', 'cast', 'lat', 'ita']
+    presel_values = ['skip', 'actual', 'recom', 'freq', 'all', 'none', 'cast', 'lat']
 
     categs = ['movie', 'tvshow', 'documentary', 'anime', 'vos', 'direct', 'torrent']
-    if config.get_setting('adult_mode') > 0: categs.append('adult')
+    if config.get_setting('adult_mode') > 0:
+        categs.append('adult')
     for c in categs:
-        preselecciones.append(config.get_localized_string(70577) + config.get_localized_category(c))
+        preselecciones.append('Modificar partiendo de %s' % config.get_localized_category(c))
         presel_values.append(c)
 
-    if item.action == 'setting_channel': # Configuración de los canales incluídos en la búsqueda
+    if item.action == 'setting_channel':  # Configuración de los canales incluídos en la búsqueda
         del preselecciones[0]
         del presel_values[0]
-    #else: # Llamada desde "buscar en otros canales" (se puede saltar la selección e ir directo a la búsqueda)
-    
+    # else: # Llamada desde "buscar en otros canales" (se puede saltar la selección e ir directo a la búsqueda)
+
     ret = platformtools.dialog_select(config.get_localized_string(59994), preselecciones)
-    if ret == -1: return False # pedido cancel
-    if presel_values[ret] == 'skip': return True # continuar sin modificar
-    elif presel_values[ret] == 'none': preselect = []
-    elif presel_values[ret] == 'all': preselect = range(len(ids))
-    elif presel_values[ret] in ['cast', 'lat']:
+    if ret == -1:
+        return False  # pedido cancel
+    if presel_values[ret] == 'skip':
+        return True  # continuar sin modificar
+    elif presel_values[ret] == 'none':
         preselect = []
-        for i, lg in enumerate(lista_lang):
-            if presel_values[ret] in lg or '*' in lg:
-                preselect.append(i)
-    elif presel_values[ret] in ['ita']:
+    elif presel_values[ret] == 'all':
+        preselect = range(len(ids))
+    elif presel_values[ret] in ['cast', 'lat']:
         preselect = []
         for i, lg in enumerate(lista_lang):
             if presel_values[ret] in lg or '*' in lg:
@@ -213,6 +404,20 @@ def setting_channel_new(item):
             channel_status = config.get_setting('include_in_global_search', canal)
             if channel_status:
                 preselect.append(i)
+
+    elif presel_values[ret] == 'recom':
+        preselect = []
+        for i, canal in enumerate(ids):
+            _not, set_canal_list = channeltools.get_channel_controls_settings(canal)
+            if set_canal_list.get('include_in_global_search', False):
+                preselect.append(i)
+
+    elif presel_values[ret] == 'freq':
+        preselect = []
+        for i, canal in enumerate(ids):
+            frequency = channeltools.get_channel_setting('frequency', canal, 0)
+            if frequency > 0:
+                preselect.append(i)
     else:
         preselect = []
         for i, ctgs in enumerate(lista_ctgs):
@@ -222,14 +427,16 @@ def setting_channel_new(item):
     # Diálogo para seleccionar
     # ------------------------
     ret = xbmcgui.Dialog().multiselect(config.get_localized_string(59994), lista, preselect=preselect, useDetails=True)
-    if ret == None: return False # pedido cancel
+    if not ret:
+        return False  # pedido cancel
     seleccionados = [ids[i] for i in ret]
 
     # Guardar cambios en canales para la búsqueda
     # -------------------------------------------
     for canal in ids:
         channel_status = config.get_setting('include_in_global_search', canal)
-        if channel_status is None: channel_status = True
+        # if not channel_status:
+        #     channel_status = True
 
         if channel_status and canal not in seleccionados:
             config.set_setting('include_in_global_search', False, canal)
@@ -238,544 +445,250 @@ def setting_channel_new(item):
 
     return True
 
-def setting_channel_old(item):
-    channels_path = os.path.join(config.get_runtime_path(), "channels", '*.json')
-    # channel_language = config.get_setting("channel_language", default="all")
-    channel_language = auto_filter()
 
-    list_controls = []
-    for infile in sorted(glob.glob(channels_path)):
-        channel_name = os.path.basename(infile)[:-5]
-        channel_parameters = channeltools.get_channel_parameters(channel_name)
+def genres_menu(item):
+    itemlist = []
+    mode = item.mode.replace('show', '')
 
-        # No incluir si es un canal inactivo
-        if not channel_parameters["active"]:
-            continue
+    genres = tmdb.get_genres(mode)
+    for key, value in genres[mode].items():
+        discovery = {'url': 'discover/%s' % mode, 'with_genres': key,
+                     'language': 'es', 'page': '1'}
 
-        # No incluir si es un canal para adultos, y el modo adulto está desactivado
-        if channel_parameters["adult"] and config.get_setting("adult_mode") == 0:
-            continue
+        itemlist.append(Item(channel=item.channel, title=value, page=1,
+                             action='discover_list', discovery=discovery,
+                             mode=item.mode))
 
-        # No incluir si el canal es en un idioma filtrado
-        if channel_language != "all" and channel_language not in str(channel_parameters["language"]) \
-                and "*" not in channel_parameters["language"]:
-            continue
-
-        # No incluir si en la configuracion del canal no existe "include_in_global_search"
-        include_in_global_search = channel_parameters["include_in_global_search"]
-
-        if not include_in_global_search:
-            continue
-        else:
-            # Se busca en la configuración del canal el valor guardado
-            include_in_global_search = config.get_setting("include_in_global_search", channel_name)
-
-        control = {'id': channel_name,
-                   'type': "bool",
-                   'label': channel_parameters["title"],
-                   'default': include_in_global_search,
-                   'enabled': True,
-                   'visible': True}
-
-        list_controls.append(control)
-
-    if config.get_setting("custom_button_value", item.channel):
-        custom_button_label = config.get_localized_string(59992)
-    else:
-        custom_button_label = config.get_localized_string(59991)
-
-    return platformtools.show_channel_settings(list_controls=list_controls,
-                                               caption=config.get_localized_string(59990),
-                                               callback="save_settings", item=item,
-                                               custom_button={'visible': True,
-                                                              'function': "cb_custom_button",
-                                                              'close': False,
-                                                              'label': custom_button_label})
+    return sorted(itemlist, key=lambda it: it.title)
 
 
-def save_settings(item, dict_values):
-    progreso = platformtools.dialog_progress(config.get_localized_string(59988), config.get_localized_string(59989))
-    n = len(dict_values)
-    for i, v in enumerate(dict_values):
-        progreso.update((i * 100) / n, config.get_localized_string(59988))
-        config.set_setting("include_in_global_search", dict_values[v], v)
-
-    progreso.close()
-    return True
-
-
-def cb_custom_button(item, dict_values):
-    value = config.get_setting("custom_button_value", item.channel)
-    if value == "":
-        value = False
-
-    for v in dict_values.keys():
-        dict_values[v] = not value
-
-    if config.set_setting("custom_button_value", not value, item.channel) == True:
-        return {"label": config.get_localized_string(59992)}
-    else:
-        return {"label": config.get_localized_string(59991)}
-
-
-def searchbycat(item):
-    # Only in xbmc/kodi
-    # Abre un cuadro de dialogo con las categorías en las que hacer la búsqueda
-
-    categories = [config.get_localized_string(30122), config.get_localized_string(30123), config.get_localized_string(30124), config.get_localized_string(30125), config.get_localized_string(59975), config.get_localized_string(59976)]
-    categories_id = ["movie", "tvshow", "anime", "documentary", "vos", "latino"]
-    list_controls = []
-    for i, category in enumerate(categories):
-        control = {'id': categories_id[i],
-                   'type': "bool",
-                   'label': category,
-                   'default': False,
-                   'enabled': True,
-                   'visible': True}
-
-        list_controls.append(control)
-    control = {'id': "separador",
-               'type': "label",
-               'label': '',
-               'default': "",
-               'enabled': True,
-               'visible': True}
-    list_controls.append(control)
-    control = {'id': "torrent",
-               'type': "bool",
-               'label': config.get_localized_string(70275),
-               'default': True,
-               'enabled': True,
-               'visible': True}
-    list_controls.append(control)
-
-    return platformtools.show_channel_settings(list_controls=list_controls, caption=config.get_localized_string(59974),
-                                               callback="search_cb", item=item)
-
-
-def search_cb(item, values=""):
-    cat = []
-    for c in values:
-        if values[c]:
-            cat.append(c)
-
-    if not len(cat):
-        return None
-    else:
-        logger.info(item.tostring())
-        logger.info(str(cat))
-        return do_search(item, cat)
-
-
-# Al llamar a esta función, el sistema pedirá primero el texto a buscar
-# y lo pasará en el parámetro "tecleado"
-def search(item, tecleado):
-    logger.info()
-    tecleado = tecleado.replace("+", " ")
-    item.category = tecleado
-
-    if tecleado != "":
-        save_search(tecleado)
-
-    if item.extra == "categorias":
-        item.extra = tecleado
-        itemlist = searchbycat(item)
-    else:
-        item.extra = tecleado
-        itemlist = do_search(item, [])
-
-    return itemlist
-
-
-def show_result(item):
-    tecleado = None
-    if item.adult and config.get_setting("adult_request_password"):
-        # Solicitar contraseña
-        tecleado = platformtools.dialog_input("", config.get_localized_string(60334), True)
-        if tecleado is None or tecleado != config.get_setting("adult_password"):
-            return []
-
-    item.channel = item.__dict__.pop('from_channel')
-    item.action = item.__dict__.pop('from_action')
-    if item.__dict__.has_key('tecleado'):
-        tecleado = item.__dict__.pop('tecleado')
-
-    # try:
-    #     channel = __import__('channels.%s' % item.channel, fromlist=["channels.%s" % item.channel])
-    # except:
-    #     import traceback
-    #     logger.error(traceback.format_exc())
-    #     return []
-
-    if tecleado:
-        # Mostrar resultados: agrupados por canales
-        return [Item().fromurl(i) for i in item.itemlist]
-        # return channel.search(item, tecleado)
-    else:
-        # Mostrar resultados: todos juntos
-        if item.infoPlus:                       #Si viene de una ventana de InfoPlus, hay que salir de esta forma...
-            del item.infoPlus                   #si no, se mete en un bucle mostrando la misma pantalla, 
-            item.title = item.title.strip()     #dando error en "handle -1"
-            return getattr(channel, item.action)(item)
-        try:
-            from platformcode import launcher
-            launcher.run(item)
-        except ImportError:
-            return getattr(channel, item.action)(item)
-
-
-def channel_search(search_results, channel_parameters, tecleado, session):
-    try:
-        exec("from channels import " + channel_parameters["channel"] + " as module")
-        mainlist = module.mainlist(Item(channel=channel_parameters["channel"]))
-        search_items = [item for item in mainlist if item.action == "search"]
-        if not search_items:
-            search_items = [Item(channel=channel_parameters["channel"], action="search")]
-
-        for item in search_items:
-            item.session = session
-            result = module.search(item.clone(), tecleado)
-            if result is None:
-                result = []
-            if len(result):
-                if not channel_parameters["title"].capitalize() in search_results:
-                    search_results[channel_parameters["title"].capitalize()] = []
-                search_results[channel_parameters['title'].capitalize()].append({"item": item,
-                                                                    "itemlist": result,
-                                                                    "thumbnail": channel_parameters["thumbnail"],
-                                                                    "adult": channel_parameters["adult"]})
-
-    except:
-        logger.error("No se puede buscar en: %s" % channel_parameters["title"])
-        import traceback
-        logger.error(traceback.format_exc())
-
-
-# Esta es la función que realmente realiza la búsqueda
-def do_search(item, categories=None):
-    logger.info("blaa categorias %s" % categories)
-
-    if item.contextual==True:
-        categories = ["Películas"]
-        setting_item = Item(channel=item.channel, title=config.get_localized_string(59994), folder=False,
-                            thumbnail=get_thumb("search.png"))
-        if not setting_channel(setting_item):
-            return False
-
-    if categories is None:
-        categories = []
-
-    multithread = config.get_setting("multithread", "search")
-    result_mode = config.get_setting("result_mode", "search")
-
-    if item.wanted!='':
-        tecleado=item.wanted
-    else:
-        tecleado = item.extra
-
+def years_menu(item):
+    import datetime
     itemlist = []
 
-    channels_path = os.path.join(config.get_runtime_path(), "channels", '*.json')
-    logger.info("channels_path=%s" % channels_path)
+    mode = item.mode.replace('show', '')
 
-    # channel_language = config.get_setting("channel_language", default="all")
-    channel_language = auto_filter()
-    logger.info("channel_language=%s" % channel_language)
+    par_year = 'primary_release_year'
 
-    # Para Kodi es necesario esperar antes de cargar el progreso, de lo contrario
-    # el cuadro de progreso queda "detras" del cuadro "cargando..." y no se le puede dar a cancelar
-    time.sleep(0.5)
-    progreso = platformtools.dialog_progress(config.get_localized_string(30993) % tecleado, "")
-    channel_files = sorted(glob.glob(channels_path), key=lambda x: os.path.basename(x))
-    import math
+    if mode != 'movie':
+        par_year = 'first_air_date_year'
 
-    threads = []
-    search_results = {}
-    start_time = time.time()
-    list_channels_search = []
+    c_year = datetime.datetime.now().year + 1
+    l_year = c_year - 31
 
-    # Extrae solo los canales a buscar
-    for index, infile in enumerate(channel_files):
-        try:
-            basename = os.path.basename(infile)
-            basename_without_extension = basename[:-5]
-            logger.info("%s..." % basename_without_extension)
+    for year in range(l_year, c_year):
+        discovery = {'url': 'discover/%s' % mode, 'page': '1',
+                     '%s' % par_year: '%s' % year,
+                     'sort_by': 'popularity.desc', 'language': 'es'}
 
-            channel_parameters = channeltools.get_channel_parameters(basename_without_extension)
-
-            # No busca si es un canal inactivo
-            if not channel_parameters["active"]:
-                logger.info("%s -no activo-" % basename_without_extension)
-                continue
-
-            # En caso de búsqueda por categorias
-            if categories:
-
-                # Si no se ha seleccionado torrent no se muestra
-                #if "torrent" not in categories and "infoPlus" not in categories:
-                #    if "torrent" in channel_parameters["categories"]:
-                #        logger.info("%s -torrent-" % basename_without_extension)
-                #        continue
-
-                for cat in categories:
-                    if cat not in channel_parameters["categories"]:
-                        logger.info("%s -no en %s-" % (basename_without_extension, cat))
-                        continue
-
-            # No busca si es un canal para adultos, y el modo adulto está desactivado
-            if channel_parameters["adult"] and config.get_setting("adult_mode") == 0:
-                logger.info("%s -adulto-" % basename_without_extension)
-                continue
-
-            # No busca si el canal es en un idioma filtrado
-            if channel_language != "all" and channel_language not in str(channel_parameters["language"]) \
-                    and "*" not in channel_parameters["language"]:
-                logger.info("%s -idioma no válido-" % basename_without_extension)
-                continue
-
-            # No busca si es un canal excluido de la búsqueda global
-            include_in_global_search = channel_parameters["include_in_global_search"]
-            if include_in_global_search:
-                # Buscar en la configuracion del canal
-                include_in_global_search = config.get_setting("include_in_global_search", basename_without_extension)
-
-            if not include_in_global_search:
-                logger.info("%s -no incluido en lista a buscar-" % basename_without_extension)
-                continue
-            list_channels_search.append(infile)
-        except:
-            logger.error("No se puede buscar en: %s" % channel_parameters["title"])
-            import traceback
-            logger.error(traceback.format_exc())
-            continue
-
-    from lib import cloudscraper
-    session = cloudscraper.create_scraper()
-
-    for index, infile in enumerate(list_channels_search):
-        try:
-           # fix float porque la division se hace mal en python 2.x
-            percentage = int(float((index+1))/len(list_channels_search)*float(100))
-            basename = os.path.basename(infile)
-            basename_without_extension = basename[:-5]
-            logger.info("%s..." % basename_without_extension)
-            channel_parameters = channeltools.get_channel_parameters(basename_without_extension)
-            # Movido aqui el progreso, para que muestre el canal exacto que está buscando
-            progreso.update(percentage,
-                            config.get_localized_string(60520) % (channel_parameters["title"]))
-            # Modo Multi Thread
-            if progreso.iscanceled():
-                progreso.close()
-                logger.info("Búsqueda cancelada")
-                return itemlist
-            if multithread:
-                t = Thread(target=channel_search, args=[search_results, channel_parameters, tecleado, session],
-                           name=channel_parameters["title"])
-                t.setDaemon(True)
-                t.start()
-                threads.append(t)
-            # Modo single Thread
-            else:
-                logger.info("Intentado búsqueda en %s de %s " % (basename_without_extension, tecleado))
-                channel_search(search_results, channel_parameters, tecleado, session)
-        except:
-            logger.error("No se puede buscar en: %s" % channel_parameters["title"])
-            import traceback
-            logger.error(traceback.format_exc())
-            continue
-
-    # Modo Multi Thread
-    # Usando isAlive() no es necesario try-except,
-    # ya que esta funcion (a diferencia de is_alive())
-    # es compatible tanto con versiones antiguas de python como nuevas
-    if multithread:
-        pendent = [a for a in threads if a.isAlive()]
-        if len(pendent) > 0: t = float(100) / len(pendent)
-        while len(pendent) > 0:
-            index = (len(threads) - len(pendent)) + 1
-            percentage = int(math.ceil(index * t))
-
-            list_pendent_names = [a.getName() for a in pendent]
-            mensaje = config.get_localized_string(70282) % (", ".join(list_pendent_names))
-            progreso.update(percentage, config.get_localized_string(60521) % (len(threads) - len(pendent) + 1, len(threads)),
-                            mensaje)
-            if progreso.iscanceled():
-                logger.info("Búsqueda cancelada")
-                break
-            time.sleep(0.5)
-            pendent = [a for a in threads if a.isAlive()]
-    total = 0
-    for channel in sorted(search_results.keys()):
-        for element in search_results[channel]:
-            total += len(element["itemlist"])
-            title = channel
-            # resultados agrupados por canales
-            if item.contextual == True or item.action == 'search_tmdb':
-                result_mode = 1
-            if result_mode == 0:
-                if len(search_results[channel]) > 1:
-                    title += " -%s" % element["item"].title.strip()
-                title = re.sub(r"\[[^\]]+\]|•", "", title)
-                title = typo(title,'bold') + typo("%02d" % len(element["itemlist"]),'_ [] color kod bold')
-                plot = config.get_localized_string(60491) + '\n' + typo('','submenu')+ '\n'
-                for i in element["itemlist"]:
-                    if type(i) == Item:
-                        plot += re.sub(r'\[(?:/)?B\]','', i.title) + '\n'
-                itemlist.append(Item(title=title, channel="search", action="show_result", url=element["item"].url,
-                                     extra=element["item"].extra, folder=True, adult=element["adult"], plot=plot,
-                                     thumbnail=element["thumbnail"], itemlist=[e.tourl() for e in element["itemlist"]],
-                                     from_action="search", from_channel=element["item"].channel, tecleado=tecleado))
-            # todos los resultados juntos, en la misma lista
-            else:
-                title = config.get_localized_string(70697) % channel
-                itemlist.append(Item(title=title, channel="search", action="",
-                                     folder=False, text_bold=True, from_channel=channel))
-                for i in element["itemlist"]:
-                    if i.action:
-                        title = "    " + i.title
-                        if "infoPlus" in categories:            #Se manrca vi viene de una ventana de InfoPlus
-                            i.infoPlus = True
-                        itemlist.append(i.clone(title=title, from_action=i.action, from_channel=i.channel,
-                                                channel="search", action="show_result", adult=element["adult"]))
-    title = config.get_localized_string(59972) % (
-    tecleado, total, time.time() - start_time)
-    itemlist.insert(0, Item(title=typo(title, 'bold color kod')))
-    progreso.close()
-    #Para opcion Buscar en otros canales
-    if item.contextual == True:
-        return exact_results(itemlist, tecleado)
-    else:
-        return itemlist
-
-
-def exact_results(results, wanted):
-    logger.info()
-    itemlist =[]
-
-    for item in results:
-        if item.action=='':
-            channel=item.from_channel
-        if item.action != '' and item.contentTitle==wanted:
-            item.title = typo(item.title,'bold') + typo(channel,'_ [] color kod bold') #'%s [%s]' % (item.title, channel)
-            itemlist.append(item)
+        itemlist.append(Item(channel=item.channel, title=str(year), action='discover_list',
+                             discovery=discovery, mode=item.mode, year_=str(year)))
+    itemlist.reverse()
+    itemlist.append(Item(channel=item.channel, title='Introduzca otro año...', url='',
+                         action="year_cus", mode=item.mode, par_year=par_year))
 
     return itemlist
 
 
-def save_search(text):
-    saved_searches_limit = int((10, 20, 30, 40,)[int(config.get_setting("saved_searches_limit", "search"))])
+def year_cus(item):
+    mode = item.mode.replace('show', '')
 
-    current_saved_searches_list = config.get_setting("saved_searches_list", "search")
-    if current_saved_searches_list is None:
-        saved_searches_list = []
-    else:
-        saved_searches_list = list(current_saved_searches_list)
-
-    if text in saved_searches_list:
-        saved_searches_list.remove(text)
-
-    saved_searches_list.insert(0, text)
-
-    config.set_setting("saved_searches_list", saved_searches_list[:saved_searches_limit], "search")
+    heading = 'Introduzca Año (4 digitos)'
+    year = platformtools.dialog_numeric(0, heading, default="")
+    item.discovery = {'url': 'discover/%s' % mode, 'page': '1',
+                      '%s' % item.par_year: '%s' % year,
+                      'sort_by': 'popularity.desc', 'language': 'es'}
+    item.action = "discover_list"
+    if year and len(year) == 4:
+        return discover_list(item)
 
 
-def clear_saved_searches(item):
-    config.set_setting("saved_searches_list", list(), "search")
-    platformtools.dialog_ok(config.get_localized_string(60329), config.get_localized_string(60424))
+def actor_list(item):
+    itemlist = []
 
+    dict_ = {'url': 'search/person', 'language': 'es', 'query': item.searched_text, 'page': item.page}
 
-def get_saved_searches():
-    current_saved_searches_list = config.get_setting("saved_searches_list", "search")
-    if current_saved_searches_list is None:
-        saved_searches_list = []
-    else:
-        saved_searches_list = list(current_saved_searches_list)
+    prof = {'Acting': 'Actor', 'Directing': 'Director', 'Production': 'Productor'}
+    plot = ''
+    item.search_type = 'person'
 
-    return saved_searches_list
+    tmdb_inf = tmdb.discovery(item, dict_=dict_)
+    results = tmdb_inf.results
+
+    if not results:
+        return results
+
+    for elem in results:
+        name = elem.get('name', '')
+        if not name:
+            continue
+
+        rol = elem.get('known_for_department', '')
+        rol = prof.get(rol, rol)
+        genero = elem.get('gender', 0)
+        if genero == 1 and rol in prof:
+            rol += 'a'
+            rol = rol.replace('Actora', 'Actriz')
+
+        know_for = elem.get('known_for', '')
+        cast_id = elem.get('id', '')
+        if know_for:
+            t_k = know_for[0].get('title', '')
+            if t_k:
+                plot = '%s en [COLOR limegreen]%s[/COLOR] y otras' % (rol, t_k)
+
+        thumbnail = 'http://image.tmdb.org/t/p/original%s' % elem.get('profile_path', '')
+        title = '%s [COLOR grey](%s)[/COLOR]' % (name, rol)
+
+        discovery = {'url': 'person/%s/combined_credits' % cast_id, 'page': '1',
+                     'sort_by': 'primary_release_date.desc', 'language': 'es'}
+
+        itemlist.append(Item(channel=item.channel, title=title, action='discover_list', cast_='cast',
+                             discovery=discovery, thumbnail=thumbnail, plot=plot, page=1))
+
+    if len(results) > 19:
+        next_ = item.page + 1
+        itemlist.append(Item(channel=item.channel, title="Pagina Siguiente >>", action='actor_list',
+                             page=next_, thumbnail=thumbnail, text_color='gold',
+                             searched_text=item.searched_text))
+    return itemlist
 
 
 def discover_list(item):
-    from platformcode import unify
+    import datetime
     itemlist = []
 
-    result = tmdb.discovery(item)
-
+    year = 0
+    tmdb_inf = tmdb.discovery(item, dict_=item.discovery, cast=item.cast_)
+    result = tmdb_inf.results
     tvshow = False
 
-    logger.debug(item)
-
     for elem in result:
-        elem['tmdb_id']=elem['id']
+        elem = tmdb_inf.get_infoLabels(elem, origen=elem)
         if 'title' in elem:
             title = unify.normalize(elem['title']).capitalize()
-            elem['year'] = scrapertools.find_single_match(elem['release_date'], r'(\d{4})-\d+-\d+')
         else:
             title = unify.normalize(elem['name']).capitalize()
             tvshow = True
+        elem['tmdb_id'] = elem['id']
 
-        new_item = Item(channel='search', title=typo(title, 'bold'), infoLabels=elem, action='do_search', extra=title,
-                        category=config.get_localized_string(70695), context ='')
+        mode = item.mode or elem['media_type']
+        thumbnail = elem.get('thumbnail', '')
+        fanart = elem.get('fanart', '')
 
-        if tvshow:
-            new_item.contentSerieName = title
-        else:
-            new_item.contentTitle = title
+        if item.cast_:
+            release = elem.get('release_date', '0000') or elem.get('first_air_date', '0000')
+            year = scrapertools.find_single_match(release, r'(\d{4})')
 
-        itemlist.append(new_item)
+        if not item.cast_ or (item.cast_ and (int(year) <= int(datetime.datetime.today().year))):
+            new_item = Item(channel='search', title=title, infoLabels=elem,
+                            action='channel_search', text=title,
+                            thumbnail=thumbnail, fanart=fanart,
+                            context='', mode=mode,
+                            release_date=year)
 
-    tmdb.set_infoLabels_itemlist(itemlist, seekTmdb=True)
+            if tvshow:
+                new_item.contentSerieName = title
+            else:
+                new_item.contentTitle = title
 
-    if item.page != '' and len(itemlist)>0:
-        next_page = str(int(item.page)+1)
-        #if not 'similar' in item.list_type:
-        #    itemlist.append(item.clone(title='Pagina Siguente', page=next_page))
-        #else:
-        itemlist.append(Item(channel=item.channel, action='discover_list', title=typo(config.get_localized_string(30992), 'color kod bold'),
-                             search_type=item.search_type, list_type=item.list_type, type=item.type, page=next_page))
+            itemlist.append(new_item)
+
+    itemlist = set_context(itemlist)
+
+    if item.cast_:
+        itemlist.sort(key=lambda it: int(it.release_date), reverse=True)
+        return itemlist
+
+    elif len(result) > 19 and item.discovery:
+        item.discovery['page'] = str(int(item.discovery['page']) + 1)
+        itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70065),
+                             list_type=item.list_type, discovery=item.discovery, text_color='gold'))
+    elif len(result) > 19:
+        next_page = str(int(item.page) + 1)
+
+        itemlist.append(Item(channel=item.channel, action='discover_list', title=config.get_localized_string(70065),
+                             list_type=item.list_type, search_type=item.search_type, mode=item.mode, page=next_page,
+                             text_color='gold'))
 
     return itemlist
 
-def search_tmdb(item):
-    logger.debug(item)
 
-    itemlist = []
-    threads = []
-    logger.debug(item)
-    wanted = item.contentTitle
-
-    search = do_search(item)
-
-    if item.contentSerieName == '':
-        results = exact_results(search, wanted)
-        for result in results:
-            logger.debug(result)
-            t = Thread(target=get_links, args=[result])
-            t.start()
-            threads.append(t)
-
-            for thread in threads:
-                thread.join()
-
-            # try:
-            #     get_links(result)
-            # except:
-            #     pass
-
-        for link in link_list:
-            if link.action == 'play' and not 'trailer' in link.title.lower() and len(itemlist) < max_links:
-                itemlist.append(link)
-
-        return sorted(itemlist, key=lambda it: it.server)
-    else:
-        for item in search:
-            if item.contentSerieName != '' and item.contentSerieName == wanted:
-                logger.debug(item)
-                itemlist.append(item)
-        return itemlist
-
-def get_links (item):
+def from_context(item):
     logger.info()
-    results =[]
-    channel = __import__('channels.%s' % item.from_channel, None, None, ["channels.%s" % item.from_channel])
-    if len(link_list) <= max_links:
-        link_list.extend(getattr(channel, item.from_action)(item))
+
+    select = setting_channel_new(item)
+
+    if not select:
+        return
+
+    if 'infoLabels' in item and 'mediatype' in item.infoLabels:
+        item.mode = item.infoLabels['mediatype']
+    else:
+        return
+
+    if 'list_type' not in item:
+        if 'wanted' in item:
+            item.title = item.wanted
+        return channel_search(item)
+
+    return discover_list(item)
+
+
+def set_context(itemlist):
+    logger.info()
+
+    for elem in itemlist:
+        elem.context = [{"title": "Elegir canales",
+                         "action": "setting_channel_new",
+                         "channel": "search"}]
+
+    return itemlist
+
+
+def send_to_temp(to_temp):
+    logger.info()
+
+    temp_path = os.path.join(config.get_data_path(), 'temp_search.json')
+    temp = dict()
+
+    for key, value in to_temp.items():
+        new_list = list()
+        for elem in value:
+            new_list.append(elem.tourl())
+        temp[key] = new_list
+
+    with open(temp_path, "w") as temp_file:
+        json.dump(temp, temp_file)
+
+
+def get_from_temp(item):
+    logger.info()
+
+    n = 30
+    nextp = n * item.page
+    prevp = n * (item.page - 1)
+
+    temp_path = os.path.join(config.get_data_path(), 'temp_search.json')
+    results = list()
+
+    with open(temp_path, "r") as temp_list:
+        from_temp = json.load(temp_list)
+
+    for elem in from_temp[item.from_channel]:
+        results.append(Item().fromurl(elem))
+
+    old_results = results
+    results = results[prevp:nextp]
+
+    if len(results) == n and len(old_results * item.page) != n:
+        results.append(Item(channel='search', title='[COLOR yellow]Pagina Siguiente >>[/COLOR]',
+                            action='get_from_temp', from_channel=item.from_channel, page=item.page + 1))
+
+    tmdb.set_infoLabels_itemlist(results, True)
+    for elem in results:
+        if not elem.infoLabels.get('year', ""):
+            elem.infoLabels['year'] = '-'
+            tmdb.set_infoLabels_item(elem, True)
+
+    return results
\ No newline at end of file