Merge remote-tracking branch 'origin/master'

core/httptools.py

@@ -56,7 +56,7 @@ HTTPTOOLS_DEFAULT_RANDOM_HEADERS = False
 #     with open(CF_LIST_PATH, "rb") as CF_File:
 #         CF_LIST = CF_File.read().splitlines()
 
-# FORCE_CLOUDSCRAPER_LIST = []
+FORCE_CLOUDSCRAPER_LIST = []
 
 def get_user_agent():
     # Returns the global user agent to be used when necessary for the url.
@@ -270,13 +270,13 @@ def downloadpage(url, **opt):
     # global CF_LIST
     CF = False
 
-    # if domain in FORCE_CLOUDSCRAPER_LIST:
-    #     from lib import cloudscraper
-    #     session = cloudscraper.create_scraper()
-    #     CF = True
-    # else:
-    from lib import requests
-    session = requests.session()
+    if domain in FORCE_CLOUDSCRAPER_LIST or opt.get('cf', False):
+        from lib import cloudscraper
+        session = cloudscraper.create_scraper()
+        CF = True
+    else:
+        from lib import requests
+        session = requests.session()
 
     # if domain in CF_LIST or opt.get('CF', False):
     if opt.get('CF', False):