domini CF dinamici (by alfa), aggiornamento url

channels.json

@@ -1,47 +1,46 @@
 {
-    "altadefinizione01": "https://altadefinizione01.photo", 
-    "altadefinizione01_link": "https://altadefinizione01.gallery", 
-    "altadefinizioneclick": "https://altadefinizione.studio", 
-    "animeforce": "https://ww1.animeforce.org", 
-    "animeleggendari": "https://animepertutti.org", 
-    "animesaturn": "https://www.animesaturn.com", 
-    "animestream": "https://www.animeworld.it", 
-    "animesubita": "http://www.animesubita.org", 
-    "animetubeita": "http://www.animetubeita.com", 
-    "animeunity": "https://www.animeunity.it", 
-    "animeworld": "https://www.animeworld.tv", 
-    "casacinema": "https://www.casacinema.me", 
-    "casacinemaInfo": "https://casacinema.life", 
-    "cb01anime": "https://www.cineblog01.network", 
-    "cineblog01": "https://cb01-nuovo-indirizzo.info", 
-    "cinemalibero": "https://cinemalibero.plus", 
-    "cinetecadibologna": "http://cinestore.cinetecadibologna.it", 
-    "dreamsub": "https://dreamsub.stream", 
-    "dsda": "https://www.dsda.press/", 
-    "fastsubita": "https://fastsubita.online", 
-    "filmgratis": "https://www.filmaltadefinizione.tv", 
-    "filmigratis": "https://filmigratis.org", 
-    "filmsenzalimiticc": "https://www.filmsenzalimiti.estate", 
-    "filmstreaming01": "https://filmstreaming01.com", 
-    "guardaserie_stream": "https://guardaserie.store", 
-    "guardaserieclick": "https://www.guardaserie.band",
-    "hd4me": "https://hd4me.net", 
-    "ilgeniodellostreaming": "https://ilgeniodellostreaming.tk",
-    "italiaserie": "https://italiaserie.org", 
-    "mondoserietv": "https://mondoserietv.com", 
-    "netfreex": "https://www.netfreex.club", 
-    "piratestreaming": "https://www.piratestreaming.online", 
-    "polpotv": "https://polpotv.club", 
-    "pufimovies": "https://pufimovies.com", 
-    "raiplay": "https://www.raiplay.it", 
-    "seriehd": "https://seriehd.net", 
-    "serietvonline": "https://serietvonline.host", 
-    "serietvsubita": "http://serietvsubita.xyz", 
-    "serietvu": "https://www.serietvu.link", 
-    "streamingaltadefinizione": "https://www.popcornstream.website/", 
-    "streamtime": "https://t.me/s/StreamTime", 
-    "tantifilm": "https://www.tantifilm.bar", 
-    "toonitalia": "https://toonitalia.org", 
-    "vedohd": "https://vedohd.uno", 
+    "altadefinizione01": "https://altadefinizione01.photo",
+    "altadefinizione01_link": "https://altadefinizione01.gallery",
+    "altadefinizioneclick": "https://altadefinizione.family",
+    "animeforce": "https://ww1.animeforce.org",
+    "animeleggendari": "https://animepertutti.org",
+    "animesaturn": "https://www.animesaturn.com",
+    "animestream": "https://www.animeworld.it",
+    "animesubita": "http://www.animesubita.org",
+    "animetubeita": "http://www.animetubeita.com",
+    "animeunity": "https://www.animeunity.it",
+    "animeworld": "https://www.animeworld.tv",
+    "casacinema": "https://www.casacinema.digital",
+    "casacinemaInfo": "https://casacinema.life",
+    "cb01anime": "https://www.cineblog01.network/",
+    "cinemalibero": "https://cinemalibero.plus",
+    "cinetecadibologna": "http://cinestore.cinetecadibologna.it",
+    "dreamsub": "https://dreamsub.stream",
+    "dsda": "https://www.dsda.press/",
+    "fastsubita": "https://fastsubita.online",
+    "filmgratis": "https://www.filmaltadefinizione.tv",
+    "filmigratis": "https://filmigratis.org",
+    "filmsenzalimiticc": "https://www.filmsenzalimiti.estate",
+    "filmstreaming01": "https://filmstreaming01.com",
+    "guardaserie_stream": "https://guardaserie.store",
+    "guardaserieclick": "https://www.guardaserie.style",
+    "hd4me": "https://hd4me.net",
+    "ilgeniodellostreaming": "https://ilgeniodellostreaming.tw",
+    "italiaserie": "https://italiaserie.org",
+    "mondoserietv": "https://mondoserietv.com",
+    "netfreex": "https://www.netfreex.club",
+    "piratestreaming": "https://www.piratestreaming.biz",
+    "polpotv": "https://polpotv.club",
+    "pufimovies": "https://pufimovies.com",
+    "raiplay": "https://www.raiplay.it",
+    "seriehd": "https://seriehd.net",
+    "serietvonline": "https://serietvonline.host",
+    "serietvsubita": "http://serietvsubita.xyz",
+    "serietvu": "https://www.serietvu.link",
+    "streamingaltadefinizione": "https://www.popcornstream.press",
+    "streamtime": "https://t.me/s/StreamTime",
+    "tantifilm": "https://www.tantifilm.pizza",
+    "toonitalia": "https://toonitalia.org",
+    "vedohd": "https://vedohd.uno",
     "vvvvid": "https://www.vvvvid.it"
 }

core/httptools.py

@@ -41,12 +41,23 @@ if HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT == 0: HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT =
 # Random use of User-Agents, if nad is not specified
 HTTPTOOLS_DEFAULT_RANDOM_HEADERS = False
 
-domainCF = list()
-channelsCF = ['guardaserieclick', 'casacinema', 'dreamsub', 'ilgeniodellostreaming', 'piratestreaming', 'altadefinizioneclick', 'altadefinizione01_link']
-otherCF = ['altadefinizione-nuovo.link', 'akvideo.stream', 'backin.net', 'vcrypt.net']
-for ch in channelsCF:
-    domainCF.append(urlparse.urlparse(config.get_channel_url(name=ch)).hostname)
-domainCF.extend(otherCF)
+# old
+# domainCF = list()
+# channelsCF = ['guardaserieclick', 'ilgeniodellostreaming']
+# otherCF = ['akvideo.stream', 'backin.net', 'vcrypt.net']
+# for ch in channelsCF:
+#     domainCF.append(urlparse.urlparse(config.get_channel_url(name=ch)).hostname)
+# domainCF.extend(otherCF)
+
+global CF_LIST
+CF_LIST = list()
+CF_LIST_PATH = os.path.join(config.get_data_path(), "CF_Domains.txt")
+
+if os.path.exists(CF_LIST_PATH):
+    with open(CF_LIST_PATH, "rb") as CF_File:
+        CF_LIST = CF_File.read().splitlines()
+
+FORCE_CLOUDSCRAPER_LIST = ['akvideo.stream']
 
 def get_user_agent():
     # Returns the global user agent to be used when necessary for the url.
@@ -257,14 +268,20 @@ def downloadpage(url, **opt):
         """
     url = scrapertools.unescape(url)
     domain = urlparse.urlparse(url).netloc
-    global domainCF
+    global CF_LIST
     CF = False
-    if domain in domainCF or opt.get('cf', False):
-        url = 'https://web.archive.org/save/' + url
-        CF = True
 
-    from lib import requests
-    session = requests.session()
+    if domain in FORCE_CLOUDSCRAPER_LIST:
+        from lib import cloudscraper
+        session = cloudscraper.create_scraper()
+        CF = True
+    else:
+        from lib import requests
+        session = requests.session()
+
+        if domain in CF_LIST or opt.get('CF', False):
+            url = 'https://web.archive.org/save/' + url
+            CF = True
 
     if config.get_setting('resolver_dns') and not opt.get('use_requests', False):
         from specials import resolverdns
@@ -382,6 +399,14 @@ def downloadpage(url, **opt):
 
     response_code = req.status_code
 
+    if req.headers.get('Server', '').startswith('cloudflare') and response_code in [429, 503, 403] and not opt.get('CF', False):
+        if domain not in CF_LIST:
+            opt["CF"] = True
+            with open(CF_LIST_PATH, "a") as CF_File:
+                CF_File.write("%s\n" % domain)
+            logger.debug("CF retry... for domain: %s" % domain)
+            return downloadpage(url, **opt)
+
     response['data'] = req.content if req.content else ''
     if CF:
         import re