stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix resolverdns

Browse Source
This commit is contained in:
marco
2020-02-21 23:28:58 +01:00
parent de6e1d8149
commit d2ba398587
4 changed files with 38 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/httptools.py
View File

@@ -259,9 +259,6 @@ def downloadpage(url, **opt):
from lib import cloudscraper
session = cloudscraper.create_scraper()
CF = True
elif opt.get('session', False):
session = opt['session'] # same session to speed up search
logger.info('same session')
else:
from lib import requests
session = requests.session()
@@ -360,6 +357,7 @@ def downloadpage(url, **opt):
timeout=opt['timeout'])
except Exception as e:
from lib import requests
req = requests.Response()
if not opt.get('ignore_response_code', False) and not proxy_data.get('stat', ''):
response['data'] = ''
response['sucess'] = False
@@ -371,7 +369,6 @@ def downloadpage(url, **opt):
show_infobox(info_dict)
return type('HTTPResponse', (), response)
else:
req = requests.Response()
req.status_code = str(e)
else:

5
core/support.py
View File

@@ -373,7 +373,7 @@ def scrape(func):
log('PATRON= ', patron)
if not data:
page = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True, session=item.session)
page = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True)
# if url may be changed and channel has findhost to update
if (not page.data or scrapertools.get_domain_from_url(page.url) != scrapertools.get_domain_from_url(item.url)) and 'findhost' in func.__globals__:
host = func.__globals__['findhost']()
@@ -382,8 +382,7 @@ def scrape(func):
jsontools.update_node(host, func.__module__.split('.')[-1], 'url')
parse[1] = scrapertools.get_domain_from_url(host)
item.url = urlparse.urlunparse(parse)
page = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True,
session=item.session)
page = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True)
data = page.data.replace("'", '"')
data = re.sub('\n|\t', ' ', data)
data = re.sub(r'>\s+<', '> <', data)