141 lines
5.2 KiB
Python
141 lines
5.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
import os
|
|
import ssl
|
|
import urlparse
|
|
|
|
from lib.requests_toolbelt.adapters import host_header_ssl
|
|
from lib import cloudscraper
|
|
from lib import doh
|
|
from platformcode import logger, config
|
|
import re
|
|
|
|
try:
|
|
import _sqlite3 as sql
|
|
except:
|
|
import sqlite3 as sql
|
|
|
|
db = os.path.join(config.get_data_path(), 'kod_db.sqlite')
|
|
|
|
|
|
class CustomSocket(ssl.SSLSocket):
|
|
def __init__(self, *args, **kwargs):
|
|
super(CustomSocket, self).__init__(*args, **kwargs)
|
|
|
|
class CustomContext(ssl.SSLContext):
|
|
def __init__(self, protocol, hostname, *args, **kwargs):
|
|
self.hostname = hostname
|
|
super(CustomContext, self).__init__(protocol)
|
|
|
|
def wrap_socket(self, sock, server_side=False,
|
|
do_handshake_on_connect=True,
|
|
suppress_ragged_eofs=True,
|
|
server_hostname=None):
|
|
return CustomSocket(sock=sock, server_side=server_side,
|
|
do_handshake_on_connect=do_handshake_on_connect,
|
|
suppress_ragged_eofs=suppress_ragged_eofs,
|
|
server_hostname=self.hostname,
|
|
_context=self)
|
|
|
|
|
|
class CipherSuiteAdapter(host_header_ssl.HostHeaderSSLAdapter, cloudscraper.CipherSuiteAdapter):
|
|
|
|
def __init__(self, hostname, *args, **kwargs):
|
|
self.cipherSuite = kwargs.get('cipherSuite', None)
|
|
self.hostname = hostname
|
|
self.ssl_context = CustomContext(ssl.PROTOCOL_TLS, hostname)
|
|
self.ssl_context.set_ciphers(self.cipherSuite)
|
|
self.ssl_context.options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1)
|
|
kwargs['ssl_context'] = self.ssl_context
|
|
|
|
cloudscraper.CipherSuiteAdapter.__init__(self, *args, **kwargs)
|
|
|
|
|
|
|
|
class session(cloudscraper.CloudScraper):
|
|
def __init__(self, *args, **kwargs):
|
|
self.conn = sql.connect(db)
|
|
self.cur = self.conn.cursor()
|
|
super(session, self).__init__(*args, **kwargs)
|
|
|
|
def getIp(self, domain):
|
|
ip = None
|
|
try:
|
|
self.cur.execute('select ip from dnscache where domain=?', (domain,))
|
|
ip = self.cur.fetchall()[0][0]
|
|
logger.info('Cache DNS: ' + domain + ' = ' + str(ip))
|
|
except:
|
|
pass
|
|
if not ip: # not cached
|
|
try:
|
|
ip = doh.query(domain)[0]
|
|
logger.info('Query DoH: ' + domain + ' = ' + str(ip))
|
|
self.writeToCache(domain, ip)
|
|
except Exception:
|
|
logger.error('Failed to resolve hostname, fallback to normal dns')
|
|
import traceback
|
|
logger.error(traceback.print_exc())
|
|
ip = domain
|
|
return ip
|
|
|
|
def writeToCache(self, domain, ip):
|
|
try:
|
|
self.cur.execute('insert into dnscache values(?,?)', (domain, ip))
|
|
except:
|
|
self.cur.execute("""CREATE TABLE IF NOT EXISTS dnscache(
|
|
"domain" TEXT NOT NULL UNIQUE,
|
|
"ip" TEXT NOT NULL,
|
|
PRIMARY KEY("domain")
|
|
);""")
|
|
self.conn.commit()
|
|
|
|
def flushDns(self, method, realUrl, domain, **kwargs):
|
|
self.cur.execute('delete from dnscache where domain=?', (domain,))
|
|
self.conn.commit()
|
|
return self.request(method, realUrl, flushedDns=True, **kwargs)
|
|
|
|
def request(self, method, url, headers=None, flushedDns=False, **kwargs):
|
|
parse = urlparse.urlparse(url)
|
|
domain = headers['Host'] if headers and 'Host' in headers.keys() else parse.netloc
|
|
ip = self.getIp(domain)
|
|
self.mount('https://', CipherSuiteAdapter(domain, cipherSuite=self.adapters['https://'].cipherSuite))
|
|
realUrl = url
|
|
|
|
if headers:
|
|
headers["Host"] = domain
|
|
else:
|
|
headers = {"Host": domain}
|
|
ret = None
|
|
tryFlush = False
|
|
|
|
parse = list(parse)
|
|
parse[1] = ip
|
|
url = urlparse.urlunparse(parse)
|
|
|
|
allow_redirects = kwargs.get('allow_redirects', True)
|
|
if 'allow_redirects' in kwargs:
|
|
del kwargs['allow_redirects']
|
|
try:
|
|
ret = super(session, self).request(method, url, headers=headers, allow_redirects=False, **kwargs)
|
|
newUrl = urlparse.urlparse(ret.headers.get('Location', realUrl))
|
|
if not newUrl.netloc and not newUrl.scheme:
|
|
newUrl = list(newUrl)
|
|
newUrl[0] = 'https://'
|
|
newUrl[1] = domain
|
|
newUrl = urlparse.urlunparse(newUrl)
|
|
if allow_redirects:
|
|
redirectN = 0
|
|
while newUrl != realUrl and redirectN < self.max_redirects:
|
|
ret = self.request(method, newUrl, headers=headers, **kwargs)
|
|
newUrl = ret.headers.get('Location', realUrl)
|
|
redirectN += 1
|
|
ret.url = newUrl
|
|
except Exception as e:
|
|
logger.info('Request for ' + domain + ' with ip ' + ip + ' failed')
|
|
logger.info(e)
|
|
tryFlush = True
|
|
if (tryFlush or not ret) and not flushedDns: # re-request ips and update cache
|
|
logger.info('Flushing dns cache for ' + domain)
|
|
return self.flushDns(method, realUrl, domain, **kwargs)
|
|
|
|
return ret
|