From c0cc4bdb2fbbb1d1152fff97b8522898c7d27ab3 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 5 May 2020 18:34:30 +0200 Subject: [PATCH] domini CF dinamici (by alfa), aggiornamento url --- channels.json | 87 +++++++++++++++++++++++------------------------ core/httptools.py | 49 +++++++++++++++++++------- 2 files changed, 80 insertions(+), 56 deletions(-) diff --git a/channels.json b/channels.json index 6ab2b6cd..3a893f1e 100644 --- a/channels.json +++ b/channels.json @@ -1,47 +1,46 @@ { - "altadefinizione01": "https://altadefinizione01.photo", - "altadefinizione01_link": "https://altadefinizione01.gallery", - "altadefinizioneclick": "https://altadefinizione.studio", - "animeforce": "https://ww1.animeforce.org", - "animeleggendari": "https://animepertutti.org", - "animesaturn": "https://www.animesaturn.com", - "animestream": "https://www.animeworld.it", - "animesubita": "http://www.animesubita.org", - "animetubeita": "http://www.animetubeita.com", - "animeunity": "https://www.animeunity.it", - "animeworld": "https://www.animeworld.tv", - "casacinema": "https://www.casacinema.me", - "casacinemaInfo": "https://casacinema.life", - "cb01anime": "https://www.cineblog01.network", - "cineblog01": "https://cb01-nuovo-indirizzo.info", - "cinemalibero": "https://cinemalibero.plus", - "cinetecadibologna": "http://cinestore.cinetecadibologna.it", - "dreamsub": "https://dreamsub.stream", - "dsda": "https://www.dsda.press/", - "fastsubita": "https://fastsubita.online", - "filmgratis": "https://www.filmaltadefinizione.tv", - "filmigratis": "https://filmigratis.org", - "filmsenzalimiticc": "https://www.filmsenzalimiti.estate", - "filmstreaming01": "https://filmstreaming01.com", - "guardaserie_stream": "https://guardaserie.store", - "guardaserieclick": "https://www.guardaserie.band", - "hd4me": "https://hd4me.net", - "ilgeniodellostreaming": "https://ilgeniodellostreaming.tk", - "italiaserie": "https://italiaserie.org", - "mondoserietv": "https://mondoserietv.com", - "netfreex": "https://www.netfreex.club", - "piratestreaming": "https://www.piratestreaming.online", - "polpotv": "https://polpotv.club", - "pufimovies": "https://pufimovies.com", - "raiplay": "https://www.raiplay.it", - "seriehd": "https://seriehd.net", - "serietvonline": "https://serietvonline.host", - "serietvsubita": "http://serietvsubita.xyz", - "serietvu": "https://www.serietvu.link", - "streamingaltadefinizione": "https://www.popcornstream.website/", - "streamtime": "https://t.me/s/StreamTime", - "tantifilm": "https://www.tantifilm.bar", - "toonitalia": "https://toonitalia.org", - "vedohd": "https://vedohd.uno", + "altadefinizione01": "https://altadefinizione01.photo", + "altadefinizione01_link": "https://altadefinizione01.gallery", + "altadefinizioneclick": "https://altadefinizione.family", + "animeforce": "https://ww1.animeforce.org", + "animeleggendari": "https://animepertutti.org", + "animesaturn": "https://www.animesaturn.com", + "animestream": "https://www.animeworld.it", + "animesubita": "http://www.animesubita.org", + "animetubeita": "http://www.animetubeita.com", + "animeunity": "https://www.animeunity.it", + "animeworld": "https://www.animeworld.tv", + "casacinema": "https://www.casacinema.digital", + "casacinemaInfo": "https://casacinema.life", + "cb01anime": "https://www.cineblog01.network/", + "cinemalibero": "https://cinemalibero.plus", + "cinetecadibologna": "http://cinestore.cinetecadibologna.it", + "dreamsub": "https://dreamsub.stream", + "dsda": "https://www.dsda.press/", + "fastsubita": "https://fastsubita.online", + "filmgratis": "https://www.filmaltadefinizione.tv", + "filmigratis": "https://filmigratis.org", + "filmsenzalimiticc": "https://www.filmsenzalimiti.estate", + "filmstreaming01": "https://filmstreaming01.com", + "guardaserie_stream": "https://guardaserie.store", + "guardaserieclick": "https://www.guardaserie.style", + "hd4me": "https://hd4me.net", + "ilgeniodellostreaming": "https://ilgeniodellostreaming.tw", + "italiaserie": "https://italiaserie.org", + "mondoserietv": "https://mondoserietv.com", + "netfreex": "https://www.netfreex.club", + "piratestreaming": "https://www.piratestreaming.biz", + "polpotv": "https://polpotv.club", + "pufimovies": "https://pufimovies.com", + "raiplay": "https://www.raiplay.it", + "seriehd": "https://seriehd.net", + "serietvonline": "https://serietvonline.host", + "serietvsubita": "http://serietvsubita.xyz", + "serietvu": "https://www.serietvu.link", + "streamingaltadefinizione": "https://www.popcornstream.press", + "streamtime": "https://t.me/s/StreamTime", + "tantifilm": "https://www.tantifilm.pizza", + "toonitalia": "https://toonitalia.org", + "vedohd": "https://vedohd.uno", "vvvvid": "https://www.vvvvid.it" } \ No newline at end of file diff --git a/core/httptools.py b/core/httptools.py index a74f431c..64cfef16 100755 --- a/core/httptools.py +++ b/core/httptools.py @@ -41,12 +41,23 @@ if HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT == 0: HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT = # Random use of User-Agents, if nad is not specified HTTPTOOLS_DEFAULT_RANDOM_HEADERS = False -domainCF = list() -channelsCF = ['guardaserieclick', 'casacinema', 'dreamsub', 'ilgeniodellostreaming', 'piratestreaming', 'altadefinizioneclick', 'altadefinizione01_link'] -otherCF = ['altadefinizione-nuovo.link', 'akvideo.stream', 'backin.net', 'vcrypt.net'] -for ch in channelsCF: - domainCF.append(urlparse.urlparse(config.get_channel_url(name=ch)).hostname) -domainCF.extend(otherCF) +# old +# domainCF = list() +# channelsCF = ['guardaserieclick', 'ilgeniodellostreaming'] +# otherCF = ['akvideo.stream', 'backin.net', 'vcrypt.net'] +# for ch in channelsCF: +# domainCF.append(urlparse.urlparse(config.get_channel_url(name=ch)).hostname) +# domainCF.extend(otherCF) + +global CF_LIST +CF_LIST = list() +CF_LIST_PATH = os.path.join(config.get_data_path(), "CF_Domains.txt") + +if os.path.exists(CF_LIST_PATH): + with open(CF_LIST_PATH, "rb") as CF_File: + CF_LIST = CF_File.read().splitlines() + +FORCE_CLOUDSCRAPER_LIST = ['akvideo.stream'] def get_user_agent(): # Returns the global user agent to be used when necessary for the url. @@ -257,14 +268,20 @@ def downloadpage(url, **opt): """ url = scrapertools.unescape(url) domain = urlparse.urlparse(url).netloc - global domainCF + global CF_LIST CF = False - if domain in domainCF or opt.get('cf', False): - url = 'https://web.archive.org/save/' + url - CF = True - from lib import requests - session = requests.session() + if domain in FORCE_CLOUDSCRAPER_LIST: + from lib import cloudscraper + session = cloudscraper.create_scraper() + CF = True + else: + from lib import requests + session = requests.session() + + if domain in CF_LIST or opt.get('CF', False): + url = 'https://web.archive.org/save/' + url + CF = True if config.get_setting('resolver_dns') and not opt.get('use_requests', False): from specials import resolverdns @@ -382,6 +399,14 @@ def downloadpage(url, **opt): response_code = req.status_code + if req.headers.get('Server', '').startswith('cloudflare') and response_code in [429, 503, 403] and not opt.get('CF', False): + if domain not in CF_LIST: + opt["CF"] = True + with open(CF_LIST_PATH, "a") as CF_File: + CF_File.write("%s\n" % domain) + logger.debug("CF retry... for domain: %s" % domain) + return downloadpage(url, **opt) + response['data'] = req.content if req.content else '' if CF: import re