httptools possibilità di utilizzare requess

This commit is contained in:
Alhaziel
2019-12-19 12:36:46 +01:00
parent fe7ebb5f22
commit 810f729bc4
2 changed files with 8 additions and 5 deletions
+7 -1
View File
@@ -395,6 +395,8 @@ def downloadpage(url, **opt):
@type ignore_response_code: bool
@return: Result of the petition
@rtype: HTTPResponse
@param use_requests: Use requests.session()
@type: bool
Parameter Type Description
-------------------------------------------------- -------------------------------------------------- ------------
@@ -422,6 +424,9 @@ def downloadpage(url, **opt):
if opt.get('session', False):
session = opt['session'] # same session to speed up search
logger.info('same session')
elif opt.get('use_requests', False):
from lib import requests
session = requests.session()
else:
from lib import cloudscraper
session = cloudscraper.create_scraper()
@@ -452,7 +457,8 @@ def downloadpage(url, **opt):
file_name = ''
opt['proxy_retries_counter'] += 1
# session.verify = False
session.verify = opt.get('verify', True)
if opt.get('cookies', True):
session.cookies = cj
session.headers.update(req_headers)