From b001731f00e45ae67a6e6fef92257f290509974d Mon Sep 17 00:00:00 2001 From: mac12m99 Date: Fri, 20 Dec 2019 23:56:59 +0100 Subject: [PATCH] support iniziale DNS over Https --- core/httptools.py | 17 ++----- lib/doh.py | 77 +++++++++++++++++++++++++++++++ specials/resolverdns.py | 100 ++++++++++++++++++++++++++++------------ 3 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 lib/doh.py diff --git a/core/httptools.py b/core/httptools.py index 136cc946..536cfa9e 100755 --- a/core/httptools.py +++ b/core/httptools.py @@ -410,24 +410,13 @@ def downloadpage(url, **opt): """ load_cookies() - - # if scrapertoolsV2.get_domain_from_url(url) in ['www.seriehd.moda', 'wstream.video', 'www.guardaserie.media', 'akvideo.stream','www.piratestreaming.top']: # cloudflare urls - # if opt.get('session', False): - # session = opt['session'] # same session to speed up search - # else: - # from lib import cloudscraper - # session = cloudscraper.create_scraper() - # else: - # from lib import requests - # session = requests.session() + from specials import resolverdns + session = resolverdns.session() if opt.get('session', False): session = opt['session'] # same session to speed up search logger.info('same session') - elif opt.get('use_requests', False): - from lib import requests - session = requests.session() - else: + elif not opt.get('use_requests', True): from lib import cloudscraper session = cloudscraper.create_scraper() diff --git a/lib/doh.py b/lib/doh.py new file mode 100644 index 00000000..93d8b12e --- /dev/null +++ b/lib/doh.py @@ -0,0 +1,77 @@ +# https://github.com/stamparm/python-doh +from __future__ import print_function + +import json +import re +import socket +import ssl +import subprocess +import sys + +PY3 = sys.version_info >= (3, 0) + +if hasattr(ssl, "_create_unverified_context"): + ssl._create_default_https_context = ssl._create_unverified_context + DOH_SERVER = "1.1.1.1" # Note: to prevent potential blocking of service based on DNS name +else: + DOH_SERVER = "cloudflare-dns.com" # Alternative servers: doh.securedns.eu, doh-de.blahdns.com, doh-jp.blahdns.com + +if PY3: + import urllib.request + _urlopen = urllib.request.urlopen + _Request = urllib.request.Request +else: + import urllib2 + _urlopen = urllib2.urlopen + _Request = urllib2.Request + +def query(name, type='A', server=DOH_SERVER, path="/dns-query", fallback=True, verbose=False): + """ + Returns domain name query results retrieved by using DNS over HTTPS protocol + # Reference: https://developers.cloudflare.com/1.1.1.1/dns-over-https/json-format/ + >>> query("one.one.one.one", fallback=False) + ['1.0.0.1', '1.1.1.1'] + >>> query("one", "NS") + ['a.nic.one.', 'b.nic.one.', 'c.nic.one.', 'd.nic.one.'] + """ + + retval = None + + try: + req = _Request("https://%s%s?name=%s&type=%s" % (server, path, name, type), headers={"Accept": "application/dns-json"}) + content = _urlopen(req).read().decode() + reply = json.loads(content) + + if "Answer" in reply: + answer = json.loads(content)["Answer"] + retval = [_["data"] for _ in answer] + else: + retval = [] + except Exception as ex: + if verbose: + print("Exception occurred: '%s'" % ex) + + if retval is None and fallback: + if type == 'A': + try: + retval = socket.gethostbyname_ex(name)[2] + except (socket.error, IndexError): + pass + + if retval is None: + process = subprocess.Popen(("nslookup", "-q=%s" % type, name), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + content = (process.communicate()[0] or "").decode().replace("\r", "") + + if "\n\n" in content and "can't" not in content.lower(): + answer = content.split("\n\n", 1)[-1] + retval = re.findall(r"(?m)^%s.+= ([^=,\n]+)$" % re.escape(name), answer) or re.findall(r"Address: (.+)", answer) + + if not retval: + match = re.search(r"Addresses: ([\s\d.]+)", answer) + if match: + retval = re.split(r"\s+", match.group(1).strip()) + + if not PY3 and retval: + retval = [_.encode() for _ in retval] + + return retval \ No newline at end of file diff --git a/specials/resolverdns.py b/specials/resolverdns.py index ba832f5a..5e147aa9 100644 --- a/specials/resolverdns.py +++ b/specials/resolverdns.py @@ -1,35 +1,77 @@ # -*- coding: utf-8 -*- -# -*- OVERRIDE RESOLVE DNS -*- +import ssl -from platformcode import config +from requests_toolbelt.adapters import host_header_ssl -if config.get_setting('resolver_dns'): - from lib import dns - from dns import resolver, name - from dns.resolver import override_system_resolver - from core import support +from lib import requests, doh +from platformcode import logger +import re - res = resolver.Resolver(configure=True) +class CustomSocket(ssl.SSLSocket): + def __init__(self, *args, **kwargs): + super(CustomSocket, self).__init__(*args, **kwargs) - #legge le impostazioni dalla configurazione e setta i relativi DNS - if config.get_setting('resolver_dns_custom') and not config.get_setting('resolver_dns_service_choose'): - res.nameservers = [config.get_setting('resolver_dns_custom1'),config.get_setting('resolver_dns_custom2')] - else: - nameservers_dns = config.get_setting('resolver_dns_service') - if nameservers_dns == 1:# 'Google' - res.nameservers = ['8.8.8.8', '2001:4860:4860::8888', - '8.8.4.4', '2001:4860:4860::8844'] - elif nameservers_dns == 2:#'OpenDns Home ip(v4)' - res.nameservers = ['208.67.222.222', '208.67.222.220'] - elif nameservers_dns == 3:#'OpenDns Family Shield ip(v4)': - res.nameservers = ['208.67.222.123', '208.67.220.123'] - elif nameservers_dns == 4:#'OpenDns ip(v6)' - #https://support.opendns.com/hc/en-us/articles/227986667-Does-OpenDNS-Support-IPv6- - res.nameservers = ['2620:119:35::35', '2620:119:53::53'] - else:#if nameservers_dns == 0:#'Cloudflare' - res.nameservers = ['1.1.1.1', '2606:4700:4700::1111', - '1.0.0.1', '2606:4700:4700::1001'] - # log di verifica dei DNS impostati, d'aiuto quando gli utenti smanettano... - support.log("NAME SERVER: {}".format(res.nameservers)) +class CustomContext(ssl.SSLContext): + def __init__(self, protocol, hostname, *args, **kwargs): + self.hostname = hostname + super(CustomContext, self).__init__(protocol) + def wrap_socket(self, sock, server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None): + return CustomSocket(sock=sock, server_side=server_side, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + server_hostname=self.hostname, + _context=self) - override_system_resolver(res) +class CipherSuiteAdapter(host_header_ssl.HostHeaderSSLAdapter): + + def __init__(self, hostname, *args, **kwargs): + self.ssl_context = kwargs.pop('ssl_context', None) + self.cipherSuite = kwargs.pop('cipherSuite', None) + self.hostname = hostname + + if not self.ssl_context: + self.ssl_context = CustomContext(ssl.PROTOCOL_TLS, hostname) + self.ssl_context.set_ciphers(self.cipherSuite) + + super(CipherSuiteAdapter, self).__init__(**kwargs) + + # ------------------------------------------------------------------------------- # + + def init_poolmanager(self, *args, **kwargs): + kwargs['ssl_context'] = self.ssl_context + return super(CipherSuiteAdapter, self).init_poolmanager(*args, **kwargs) + + # ------------------------------------------------------------------------------- # + + def proxy_manager_for(self, *args, **kwargs): + kwargs['ssl_context'] = self.ssl_context + return super(CipherSuiteAdapter, self).proxy_manager_for(*args, **kwargs) + +# ------------------------------------------------------------------------------- # + +class session(requests.Session): + def request(self, method, url, + params=None, data=None, headers=None, cookies=None, files=None, + auth=None, timeout=None, allow_redirects=True, proxies=None, + hooks=None, stream=None, verify=None, cert=None, json=None): + protocol, domain, port, resource = re.match('^(http[s]?:\/\/)?([^:\/\s]+)(?::([^\/]*))?([^\s]+)$', url, flags=re.IGNORECASE).groups() + self.mount('https://', CipherSuiteAdapter(domain, cipherSuite="ALL")) + try: + ip = doh.query(domain) + logger.info('Query DoH: ' + domain + ' = ' + str(ip)) + url = protocol + ip[0] + (':' + port if port else '') + resource + except Exception: + logger.error('Failed to resolve hostname, fallback to normal dns') + import traceback + logger.error(traceback.print_exc()) + if headers: + headers["Host"] = domain + else: + headers = {"Host": domain} + return super(session, self).request(method, url, + params, data, headers, cookies, files, + auth, timeout, allow_redirects, proxies, + hooks, stream, verify, cert, json)