stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

rimosso proxytools

Browse Source
This commit is contained in:
marco
2019-12-28 16:34:49 +01:00
parent dfdb150130
commit 1ab7044d22
3 changed files with 101 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/httptools.py
+101 -274
View File
@@ -144,34 +144,6 @@ def random_useragent():
return default_headers["User-Agent"] return default_headers["User-Agent"]
def channel_proxy_list(url, forced_proxy=None):
import base64
import ast
try:
proxy_channel_bloqued_str = base64.b64decode(config.get_setting
('proxy_channel_bloqued')).decode('utf-8')
proxy_channel_bloqued = dict()
proxy_channel_bloqued = ast.literal_eval(proxy_channel_bloqued_str)
except:
logger.debug('Proxytools not initialized correctly')
return False
if not url.endswith('/'):
url += '/'
if scrapertools.find_single_match(url, r'(?:http.*\:)?\/\/(?:www\.)?([^\?|\/]+)(?:\?|\/)') \
in proxy_channel_bloqued:
if forced_proxy and forced_proxy not in ['Total', 'ProxyDirect', 'ProxyCF', 'ProxyWeb']:
if forced_proxy in proxy_channel_bloqued[scrapertools.find_single_match(url, r'(?:http.*\:)?\/\/(?:www\.)?([^\?|\/]+)(?:\?|\/)')]:
return True
else:
return False
if forced_proxy:
return True
if not 'OFF' in proxy_channel_bloqued[scrapertools.find_single_match(url, r'(?:http.*\:)?\/\/(?:www\.)?([^\?|\/]+)(?:\?|\/)')]:
return True
return False
def show_infobox(info_dict): def show_infobox(info_dict):
logger.info() logger.info()
@@ -232,137 +204,6 @@ def show_infobox(info_dict):
logger.info('%s%s%s' % (box['r_dn_corner'], box['fill'] * width, box['l_dn_corner'])) logger.info('%s%s%s' % (box['r_dn_corner'], box['fill'] * width, box['l_dn_corner']))
return return
def check_proxy(url, **opt):
proxy_data = dict()
proxy_data['dict'] = {}
proxy = opt.get('proxy', True)
proxy_web = opt.get('proxy_web', False)
proxy_addr_forced = opt.get('proxy_addr_forced', None)
forced_proxy = opt.get('forced_proxy', None)
try:
if (proxy or proxy_web) and (forced_proxy or proxy_addr_forced or
channel_proxy_list(url, forced_proxy=forced_proxy)):
import proxytools
proxy_data['addr'], proxy_data['CF_addr'], proxy_data['web_name'], \
proxy_data['log'] = proxytools.get_proxy_addr(url, post=opt.get('post', None), forced_proxy=forced_proxy)
if proxy_addr_forced and proxy_data['log']:
proxy_data['log'] = scrapertools.find_single_match(str(proxy_addr_forced), r"{'http.*':\s*'(.*?)'}")
if proxy and proxy_data['addr']:
if proxy_addr_forced: proxy_data['addr'] = proxy_addr_forced
proxy_data['dict'] = proxy_data['addr']
proxy_data['stat'] = ', Proxy Direct ' + proxy_data['log']
elif proxy and proxy_data['CF_addr']:
if proxy_addr_forced: proxy_data['CF_addr'] = proxy_addr_forced
proxy_data['dict'] = proxy_data['CF_addr']
proxy_data['stat'] = ', Proxy CF ' + proxy_data['log']
elif proxy and proxy_addr_forced:
proxy_data['addr'] = proxy_addr_forced
proxy_data['dict'] = proxy_data['addr']
proxy_data['stat'] = ', Proxy Direct ' + proxy_data['log']
elif proxy and not proxy_data['addr'] and not proxy_data['CF_addr'] \
and not proxy_addr_forced:
proxy = False
if not proxy_data['web_name']:
proxy_data['addr'], proxy_data['CF_addr'], proxy_data['web_name'], \
proxy_data['log'] = proxytools.get_proxy_addr(url, forced_proxy='Total')
if proxy_data['web_name']:
proxy_web = True
else:
proxy_web = False
if proxy_data['addr']:
proxy = True
proxy_data['dict'] = proxy_data['addr']
proxy_data['stat'] = ', Proxy Direct ' + proxy_data['log']
if proxy_web and proxy_data['web_name']:
if opt.get('post', None): proxy_data['log'] = '(POST) ' + proxy_data['log']
url, opt['post'], headers_proxy, proxy_data['web_name'] = \
proxytools.set_proxy_web(url, proxy_data['web_name'], post=opt.get('post', None))
if proxy_data['web_name']:
proxy_data['stat'] = ', Proxy Web ' + proxy_data['log']
if headers_proxy:
request_headers.update(dict(headers_proxy))
if proxy_web and not proxy_data['web_name']:
proxy_web = False
proxy_data['addr'], proxy_data['CF_addr'], proxy_data['web_name'], \
proxy_data['log'] = proxytools.get_proxy_addr(url, forced_proxy='Total')
if proxy_data['CF_addr']:
proxy = True
proxy_data['dict'] = proxy_data['CF_addr']
proxy_data['stat'] = ', Proxy CF ' + proxy_data['log']
elif proxy_data['addr']:
proxy = True
proxy_data['dict'] = proxy_data['addr']
proxy_data['stat'] = ', Proxy Direct ' + proxy_data['log']
except:
import traceback
logger.error(traceback.format_exc())
opt['proxy'] = ''
opt['proxy_web'] = ''
proxy_data['stat'] = ''
proxy_data['addr'] = ''
proxy_data['CF_addr'] = ''
proxy_data['dict'] = {}
proxy_data['web_name'] = ''
proxy_data['log'] = ''
url = opt['url_save']
try:
proxy_data['addr']['https'] = str('https://'+ proxy_data['addr']['https'])
except:
pass
return url, proxy_data, opt
def proxy_post_processing(url, proxy_data, response, opt):
opt['out_break'] = False
try:
if ', Proxy Web' in proxy_data.get('stat', ''):
import proxytools
response["data"] = proxytools.restore_after_proxy_web(response["data"],
proxy_data['web_name'], opt['url_save'])
if response["data"] == 'ERROR':
response['sucess'] = False
if response["code"] == 302:
proxy_data['stat'] = ', Proxy Direct'
opt['forced_proxy'] = 'ProxyDirect'
url = opt['url_save']
opt['post'] = opt['post_save']
response['sucess'] = False
if proxy_data.get('stat', '') and response['sucess'] == False and \
opt.get('proxy_retries_counter', 0) <= opt.get('proxy_retries', 1) and opt.get('count_retries_tot', 5) > 1:
import proxytools
if ', Proxy Direct' in proxy_data.get('stat', ''):
proxytools.get_proxy_list_method(proxy_init='ProxyDirect',
error_skip=proxy_data['addr'], url_test=url)
elif ', Proxy CF' in proxy_data.get('stat', ''):
proxytools.get_proxy_list_method(proxy_init='ProxyCF',
error_skip=proxy_data['CF_addr'])
url = opt['url_save']
elif ', Proxy Web' in proxy_data.get('stat', ''):
if channel_proxy_list(opt['url_save'], forced_proxy=proxy_data['web_name']):
opt['forced_proxy'] = 'ProxyCF'
url =opt['url_save']
opt['post'] = opt['post_save']
else:
proxytools.get_proxy_list_method(proxy_init='ProxyWeb',
error_skip=proxy_data['web_name'])
url =opt['url_save']
opt['post'] = opt['post_save']
else:
opt['out_break'] = True
except:
import traceback
logger.error(traceback.format_exc())
opt['out_break'] = True
return response["data"], response['sucess'], url, opt
def downloadpage(url, **opt): def downloadpage(url, **opt):
@@ -437,147 +278,133 @@ def downloadpage(url, **opt):
req_headers['User-Agent'] = random_useragent() req_headers['User-Agent'] = random_useragent()
url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]") url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
opt['proxy_retries_counter'] = 0
opt['url_save'] = url opt['url_save'] = url
opt['post_save'] = opt.get('post', None) opt['post_save'] = opt.get('post', None)
while opt['proxy_retries_counter'] < opt.get('proxy_retries', 1): response = {}
response = {} info_dict = []
info_dict = [] payload = dict()
payload = dict() files = {}
files = {} file_name = ''
file_name = ''
opt['proxy_retries_counter'] += 1
session.verify = opt.get('verify', True) session.verify = opt.get('verify', True)
if opt.get('cookies', True): if opt.get('cookies', True):
session.cookies = cj session.cookies = cj
session.headers.update(req_headers) session.headers.update(req_headers)
# Prepare the url in case you need a proxy, or if proxies are sent from the channel proxy_data = {'dict': {}}
# url, proxy_data, opt = check_proxy(url, **opt)
# if opt.get('proxies', None) is not None:
# session.proxies = opt['proxies']
# elif proxy_data.get('dict', {}):
# session.proxies = proxy_data['dict']
proxy_data = {'dict': {}}
inicio = time.time() inicio = time.time()
if opt.get('timeout', None) is None and HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT is not None: if opt.get('timeout', None) is None and HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT is not None:
opt['timeout'] = HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT opt['timeout'] = HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT
if opt['timeout'] == 0: opt['timeout'] = None if opt['timeout'] == 0: opt['timeout'] = None
if len(url) > 0: if len(url) > 0:
try: try:
if opt.get('post', None) is not None or opt.get('file', None) is not None: if opt.get('post', None) is not None or opt.get('file', None) is not None:
if opt.get('post', None) is not None: if opt.get('post', None) is not None:
# Convert string post in dict # Convert string post in dict
try: try:
json.loads(opt['post']) json.loads(opt['post'])
payload = opt['post'] payload = opt['post']
except: except:
if not isinstance(opt['post'], dict): if not isinstance(opt['post'], dict):
post = urlparse.parse_qs(opt['post'], keep_blank_values=1) post = urlparse.parse_qs(opt['post'], keep_blank_values=1)
payload = dict() payload = dict()
for key, value in post.items(): for key, value in post.items():
try: try:
payload[key] = value[0] payload[key] = value[0]
except: except:
payload[key] = '' payload[key] = ''
else:
payload = opt['post']
# Verify 'file' and 'file_name' options to upload a buffer or file
if opt.get('file', None) is not None:
if os.path.isfile(opt['file']):
if opt.get('file_name', None) is None:
path_file, opt['file_name'] = os.path.split(opt['file'])
files = {'file': (opt['file_name'], open(opt['file'], 'rb'))}
file_name = opt['file']
else: else:
files = {'file': (opt.get('file_name', 'Default'), opt['file'])} payload = opt['post']
file_name = opt.get('file_name', 'Default') + ', Buffer de memoria'
info_dict = fill_fields_pre(url, opt, proxy_data, file_name) # Verify 'file' and 'file_name' options to upload a buffer or file
if opt.get('only_headers', False): if opt.get('file', None) is not None:
# Makes the request with HEAD method if os.path.isfile(opt['file']):
req = session.head(url, allow_redirects=opt.get('follow_redirects', True), if opt.get('file_name', None) is None:
timeout=opt['timeout']) path_file, opt['file_name'] = os.path.split(opt['file'])
files = {'file': (opt['file_name'], open(opt['file'], 'rb'))}
file_name = opt['file']
else: else:
# Makes the request with POST method files = {'file': (opt.get('file_name', 'Default'), opt['file'])}
req = session.post(url, data=payload, allow_redirects=opt.get('follow_redirects', True), file_name = opt.get('file_name', 'Default') + ', Buffer de memoria'
files=files, timeout=opt['timeout'])
elif opt.get('only_headers', False): info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
info_dict = fill_fields_pre(url, opt, proxy_data, file_name) if opt.get('only_headers', False):
# Makes the request with HEAD method # Makes the request with HEAD method
req = session.head(url, allow_redirects=opt.get('follow_redirects', True), req = session.head(url, allow_redirects=opt.get('follow_redirects', True),
timeout=opt['timeout']) timeout=opt['timeout'])
else: else:
info_dict = fill_fields_pre(url, opt, proxy_data, file_name) # Makes the request with POST method
# Makes the request with GET method req = session.post(url, data=payload, allow_redirects=opt.get('follow_redirects', True),
req = session.get(url, allow_redirects=opt.get('follow_redirects', True), files=files, timeout=opt['timeout'])
timeout=opt['timeout'])
except Exception as e:
from lib import requests
if not opt.get('ignore_response_code', False) and not proxy_data.get('stat', ''):
req = requests.Response()
response['data'] = ''
response['sucess'] = False
info_dict.append(('Success', 'False'))
response['code'] = str(e)
info_dict.append(('Response code', str(e)))
info_dict.append(('Finalizado en', time.time() - inicio))
if not opt.get('alfa_s', False):
show_infobox(info_dict)
return type('HTTPResponse', (), response)
else:
req = requests.Response()
req.status_code = str(e)
else: elif opt.get('only_headers', False):
response['data'] = '' info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
response['sucess'] = False # Makes the request with HEAD method
response['code'] = '' req = session.head(url, allow_redirects=opt.get('follow_redirects', True),
return type('HTTPResponse', (), response) timeout=opt['timeout'])
else:
info_dict = fill_fields_pre(url, opt, proxy_data, file_name)
# Makes the request with GET method
req = session.get(url, allow_redirects=opt.get('follow_redirects', True),
timeout=opt['timeout'])
except Exception as e:
from lib import requests
if not opt.get('ignore_response_code', False) and not proxy_data.get('stat', ''):
req = requests.Response()
response['data'] = ''
response['sucess'] = False
info_dict.append(('Success', 'False'))
response['code'] = str(e)
info_dict.append(('Response code', str(e)))
info_dict.append(('Finalizado en', time.time() - inicio))
if not opt.get('alfa_s', False):
show_infobox(info_dict)
return type('HTTPResponse', (), response)
else:
req = requests.Response()
req.status_code = str(e)
response_code = req.status_code else:
response['data'] = ''
response['sucess'] = False
response['code'] = ''
return type('HTTPResponse', (), response)
response['data'] = req.content response_code = req.status_code
response['url'] = req.url
if not response['data']:
response['data'] = ''
try:
response['json'] = to_utf8(req.json())
except:
response['json'] = dict()
response['code'] = response_code
response['headers'] = req.headers
response['cookies'] = req.cookies
info_dict, response = fill_fields_post(info_dict, req, response, req_headers, inicio) response['data'] = req.content
response['url'] = req.url
if not response['data']:
response['data'] = ''
try:
response['json'] = to_utf8(req.json())
except:
response['json'] = dict()
response['code'] = response_code
response['headers'] = req.headers
response['cookies'] = req.cookies
if opt.get('cookies', True): info_dict, response = fill_fields_post(info_dict, req, response, req_headers, inicio)
save_cookies(alfa_s=opt.get('alfa_s', False))
# is_channel = inspect.getmodule(inspect.currentframe().f_back) if opt.get('cookies', True):
# is_channel = scrapertools.find_single_match(str(is_channel), "<module '(channels).*?'") save_cookies(alfa_s=opt.get('alfa_s', False))
# if is_channel and isinstance(response_code, int):
# if not opt.get('ignore_response_code', False) and not proxy_data.get('stat', ''):
# if response_code > 399:
# show_infobox(info_dict)
# raise WebErrorException(urlparse.urlparse(url)[1])
if not 'api.themoviedb' in url and not opt.get('alfa_s', False): # is_channel = inspect.getmodule(inspect.currentframe().f_back)
show_infobox(info_dict) # is_channel = scrapertools.find_single_match(str(is_channel), "<module '(channels).*?'")
# if is_channel and isinstance(response_code, int):
# if not opt.get('ignore_response_code', False) and not proxy_data.get('stat', ''):
# if response_code > 399:
# show_infobox(info_dict)
# raise WebErrorException(urlparse.urlparse(url)[1])
# If there is a proxy error, refresh the list and retry the number indicated in proxy_retries if not 'api.themoviedb' in url and not opt.get('alfa_s', False):
# response['data'], response['sucess'], url, opt = proxy_post_processing(url, proxy_data, response, opt) show_infobox(info_dict)
# if opt.get('out_break', False):
# break
return type('HTTPResponse', (), response) return type('HTTPResponse', (), response)
core/proxytools.py
-1
View File
File diff suppressed because one or more lines are too long
videolibrary_service.py
-4
View File
@@ -381,15 +381,11 @@ if __name__ == "__main__":
if not config.dev_mode(): if not config.dev_mode():
from platformcode import updater from platformcode import updater
updater.check_addon_init() updater.check_addon_init()
# get_channel_json() -> disabilitato, lo si fa con l'updater
# Copia Custom code a las carpetas de Alfa desde la zona de Userdata # Copia Custom code a las carpetas de Alfa desde la zona de Userdata
from platformcode import custom_code from platformcode import custom_code
custom_code.init() custom_code.init()
# Identifica la dirección Proxy y la lista de alternativas
# from core import proxytools
# proxytools.get_proxy_list()
if not config.get_setting("update", "videolibrary") == 2: if not config.get_setting("update", "videolibrary") == 2:
check_for_update(overwrite=False) check_for_update(overwrite=False)