From c5cb55bf5e8eaf9b9c80ce9a93966fb1dfcb0ebf Mon Sep 17 00:00:00 2001 From: Alhaziel Date: Mon, 13 Jan 2020 16:05:53 +0100 Subject: [PATCH 1/4] Fix Locandine e trame non Visibili --- core/tmdb.py | 51 +++++++++++++++++------------------ resources/settings.xml | 60 ++---------------------------------------- 2 files changed, 28 insertions(+), 83 deletions(-) diff --git a/core/tmdb.py b/core/tmdb.py index 65c6ff0b..a369f6d0 100644 --- a/core/tmdb.py +++ b/core/tmdb.py @@ -160,7 +160,7 @@ def cache_response(fn): conn = sqlite3.connect(fname, timeout=15) c = conn.cursor() url = re.sub('&year=-', '', args[0]) - logger.error('la url %s' % url) + # logger.error('la url %s' % url) url_base64 = base64.b64encode(url) c.execute("SELECT response, added FROM tmdb_cache WHERE url=?", (url_base64,)) row = c.fetchone() @@ -185,7 +185,7 @@ def cache_response(fn): # error al obtener los datos except Exception, ex: message = "An exception of type %s occured. Arguments:\n%s" % (type(ex).__name__, repr(ex.args)) - logger.error("error en: %s" % message) + logger.error("error in: %s" % message) return result @@ -214,10 +214,10 @@ def set_infoLabels(source, seekTmdb=True, idioma_busqueda=def_lang, forced=False start_time = time.time() if type(source) == list: ret = set_infoLabels_itemlist(source, seekTmdb, idioma_busqueda) - logger.debug("Se han obtenido los datos de %i enlaces en %f segundos" % (len(source), time.time() - start_time)) + logger.debug("The data of %i links were obtained in %f seconds" % (len(source), time.time() - start_time)) else: ret = set_infoLabels_item(source, seekTmdb, idioma_busqueda) - logger.debug("Se han obtenido los datos del enlace en %f segundos" % (time.time() - start_time)) + logger.debug("The data of %i links were obtained in %f seconds" % (time.time() - start_time)) return ret @@ -242,6 +242,7 @@ def set_infoLabels_itemlist(item_list, seekTmdb=False, idioma_busqueda=def_lang, negativo en caso contrario. @rtype: list """ + if not config.get_setting('tmdb_active') and not forced: return import threading @@ -314,7 +315,7 @@ def set_infoLabels_item(item, seekTmdb=True, idioma_busqueda=def_lang, lock=None try: numtemporada = int(item.infoLabels['season']) except ValueError: - logger.debug("El numero de temporada no es valido") + logger.debug("The season number is not valid.") return -1 * len(item.infoLabels) if lock: @@ -340,7 +341,7 @@ def set_infoLabels_item(item, seekTmdb=True, idioma_busqueda=def_lang, lock=None try: episode = int(item.infoLabels['episode']) except ValueError: - logger.debug("El número de episodio (%s) no es valido" % repr(item.infoLabels['episode'])) + logger.debug("The episode number (%s) is not valid" % repr(item.infoLabels['episode'])) return -1 * len(item.infoLabels) # Tenemos numero de temporada y numero de episodio validos... @@ -847,7 +848,7 @@ class Tmdb(object): self.__discover() else: - logger.debug("Creado objeto vacio") + logger.debug("Created empty object") @staticmethod @cache_response @@ -879,7 +880,7 @@ class Tmdb(object): # error al obtener los datos except Exception, ex: message = "An exception of type %s occured. Arguments:\n%s" % (type(ex).__name__, repr(ex.args)) - logger.error("error en: %s" % message) + logger.error("error in: %s" % message) dict_data = {} return dict_data @@ -895,7 +896,7 @@ class Tmdb(object): url = ('http://api.themoviedb.org/3/genre/%s/list?api_key=a1ab8b8669da03637a4b98fa39c39228&language=%s' % (tipo, idioma)) try: - logger.info("[Tmdb.py] Rellenando dicionario de generos") + logger.info("[Tmdb.py] Filling in dictionary of genres") resultado = cls.get_json(url) lista_generos = resultado["genres"] @@ -903,7 +904,7 @@ class Tmdb(object): for i in lista_generos: cls.dic_generos[idioma][tipo][str(i["id"])] = i["name"] except: - logger.error("Error generando diccionarios") + logger.error("Error generating dictionaries") def __by_id(self, source='tmdb'): @@ -923,7 +924,7 @@ class Tmdb(object): '&language=%s' % (self.busqueda_id, source, self.busqueda_idioma)) buscando = "%s: %s" % (source.capitalize(), self.busqueda_id) - logger.info("[Tmdb.py] Buscando %s:\n%s" % (buscando, url)) + logger.info("[Tmdb.py] Searching %s:\n%s" % (buscando, url)) resultado = self.get_json(url) if resultado: @@ -940,8 +941,8 @@ class Tmdb(object): else: # No hay resultados de la busqueda - msg = "La busqueda de %s no dio resultados." % buscando - logger.debug(msg) + msg = "The search of %s gave no results" % buscando + # logger.debug(msg) def __search(self, index_results=0, page=1): self.result = ResultDictDefault() @@ -963,7 +964,7 @@ class Tmdb(object): url += '&year=%s' % self.busqueda_year buscando = self.busqueda_texto.capitalize() - logger.info("[Tmdb.py] Buscando %s en pagina %s:\n%s" % (buscando, page, url)) + logger.info("[Tmdb.py] Searching %s on page %s:\n%s" % (buscando, page, url)) resultado = self.get_json(url) total_results = resultado.get("total_results", 0) @@ -984,7 +985,7 @@ class Tmdb(object): if index_results >= len(results): # Se ha solicitado un numero de resultado mayor de los obtenidos logger.error( - "La busqueda de '%s' dio %s resultados para la pagina %s\nImposible mostrar el resultado numero %s" + "The search for '%s' gave %s results for the page %s \n It is impossible to show the result number %s" % (buscando, len(results), page, index_results)) return 0 @@ -997,7 +998,7 @@ class Tmdb(object): else: # No hay resultados de la busqueda - msg = "La busqueda de '%s' no dio resultados para la pagina %s" % (buscando, page) + msg = "The search for '%s' gave no results for page %s" % (buscando, page) logger.error(msg) return 0 @@ -1021,7 +1022,7 @@ class Tmdb(object): url = ('http://api.themoviedb.org/3/%s?api_key=a1ab8b8669da03637a4b98fa39c39228&%s' % (type_search, "&".join(params))) - logger.info("[Tmdb.py] Buscando %s:\n%s" % (type_search, url)) + logger.info("[Tmdb.py] Searcing %s:\n%s" % (type_search, url)) resultado = self.get_json(url) total_results = resultado.get("total_results", -1) @@ -1045,7 +1046,7 @@ class Tmdb(object): if index_results >= len(results): logger.error( - "La busqueda de '%s' no dio %s resultados" % (type_search, index_results)) + "The search for '%s' did not give %s results" % (type_search, index_results)) return 0 # Retornamos el numero de resultados de esta pagina @@ -1061,7 +1062,7 @@ class Tmdb(object): return len(self.results) else: # No hay resultados de la busqueda - logger.error("La busqueda de '%s' no dio resultados" % type_search) + logger.error("The search for '%s' gave no results" % type_search) return 0 def load_resultado(self, index_results=0, page=1): @@ -1311,20 +1312,20 @@ class Tmdb(object): url = "http://api.themoviedb.org/3/tv/%s/season/%s?api_key=a1ab8b8669da03637a4b98fa39c39228&language=%s" \ "&append_to_response=credits" % (self.result["id"], numtemporada, self.busqueda_idioma) - buscando = "id_Tmdb: " + str(self.result["id"]) + " temporada: " + str(numtemporada) + "\nURL: " + url - logger.info("[Tmdb.py] Buscando " + buscando) + buscando = "id_Tmdb: " + str(self.result["id"]) + " season: " + str(numtemporada) + "\nURL: " + url + logger.info("[Tmdb.py] Searcing " + buscando) try: self.temporada[numtemporada] = self.get_json(url) except: - logger.error("No se ha podido obtener la temporada") + logger.error("Unable to get the season") self.temporada[numtemporada] = {"status_code": 15, "status_message": "Failed"} self.temporada[numtemporada] = {"episodes": {}} if "status_code" in self.temporada[numtemporada]: #Se ha producido un error msg = config.get_localized_string(70496) + buscando + config.get_localized_string(70497) - msg += "\nError de tmdb: %s %s" % ( + msg += "\nTmdb error: %s %s" % ( self.temporada[numtemporada]["status_code"], self.temporada[numtemporada]["status_message"]) logger.debug(msg) self.temporada[numtemporada] = {} @@ -1351,7 +1352,7 @@ class Tmdb(object): capitulo = int(capitulo) numtemporada = int(numtemporada) except ValueError: - logger.debug("El número de episodio o temporada no es valido") + logger.debug("The episode or season number is not valid") return {} temporada = self.get_temporada(numtemporada) @@ -1361,7 +1362,7 @@ class Tmdb(object): if len(temporada["episodes"]) == 0 or len(temporada["episodes"]) < capitulo: # Se ha producido un error - logger.error("Episodio %d de la temporada %d no encontrado." % (capitulo, numtemporada)) + logger.error("Episode %d of the season %d not found." % (capitulo, numtemporada)) return {} ret_dic = dict() diff --git a/resources/settings.xml b/resources/settings.xml index 2a2adbbc..d225c417 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -77,61 +77,8 @@ - + @@ -144,6 +91,7 @@ + @@ -162,9 +110,6 @@ - @@ -179,5 +124,4 @@ - From 7e4371f482fe30cd27fbb2865a8eca6cef165069 Mon Sep 17 00:00:00 2001 From: Alhaziel Date: Mon, 13 Jan 2020 16:06:40 +0100 Subject: [PATCH 2/4] Aggiornamento Cloudscraper --- lib/cloudscraper/__init__.py | 70 +++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/lib/cloudscraper/__init__.py b/lib/cloudscraper/__init__.py index 541b32cf..f4e584e5 100644 --- a/lib/cloudscraper/__init__.py +++ b/lib/cloudscraper/__init__.py @@ -1,4 +1,3 @@ -# https://github.com/VeNoMouS/cloudscraper import logging import re import sys @@ -38,7 +37,7 @@ except ImportError: # ------------------------------------------------------------------------------- # -__version__ = '1.2.16' +__version__ = '1.2.19' # ------------------------------------------------------------------------------- # @@ -163,7 +162,6 @@ class CloudScraper(Session): def request(self, method, url, *args, **kwargs): # pylint: disable=E0203 - if kwargs.get('proxies') and kwargs.get('proxies') != self.proxies: self.proxies = kwargs.get('proxies') @@ -198,6 +196,7 @@ class CloudScraper(Session): else: if not resp.is_redirect and resp.status_code not in [429, 503]: self._solveDepthCnt = 0 + return resp # ------------------------------------------------------------------------------- # @@ -243,7 +242,7 @@ class CloudScraper(Session): return False # ------------------------------------------------------------------------------- # - # check if the response contains a valid Cloudflare reCaptcha challenge + # check if the response contains Firewall 1020 Error # ------------------------------------------------------------------------------- # @staticmethod @@ -270,10 +269,7 @@ class CloudScraper(Session): def is_Challenge_Request(self, resp): if self.is_Firewall_Blocked(resp): sys.tracebacklimit = 0 - raise RuntimeError( - 'Cloudflare has a restriction on your IP (Code 1020 Detected), ' - 'you are BLOCKED.' - ) + raise RuntimeError('Cloudflare has blocked this request (Code 1020 Detected).') if self.is_reCaptcha_Challenge(resp) or self.is_IUAM_Challenge(resp): return True @@ -434,6 +430,7 @@ class CloudScraper(Session): # ------------------------------------------------------------------------------- # if submit_url: + def updateAttr(obj, name, newValue): try: obj[name].update(newValue) @@ -450,13 +447,18 @@ class CloudScraper(Session): 'data', submit_url['data'] ) + + urlParsed = urlparse(resp.url) cloudflare_kwargs['headers'] = updateAttr( cloudflare_kwargs, 'headers', - {'Referer': resp.url} + { + 'Origin': '{}://{}'.format(urlParsed.scheme, urlParsed.netloc), + 'Referer': resp.url + } ) - ret = self.request( + challengeSubmitResponse = self.request( 'POST', submit_url['url'], **cloudflare_kwargs @@ -464,13 +466,44 @@ class CloudScraper(Session): # ------------------------------------------------------------------------------- # # Return response if Cloudflare is doing content pass through instead of 3xx + # else request with redirect URL also handle protocol scheme change http -> https # ------------------------------------------------------------------------------- # - if not ret.is_redirect: - return ret + if not challengeSubmitResponse.is_redirect: + return challengeSubmitResponse + else: + cloudflare_kwargs = deepcopy(kwargs) + + if not urlparse(challengeSubmitResponse.headers['Location']).netloc: + cloudflare_kwargs['headers'] = updateAttr( + cloudflare_kwargs, + 'headers', + {'Referer': '{}://{}'.format(urlParsed.scheme, urlParsed.netloc)} + ) + return self.request( + resp.request.method, + '{}://{}{}'.format( + urlParsed.scheme, + urlParsed.netloc, + challengeSubmitResponse.headers['Location'] + ), + **cloudflare_kwargs + ) + else: + redirectParsed = urlparse(challengeSubmitResponse.headers['Location']) + cloudflare_kwargs['headers'] = updateAttr( + cloudflare_kwargs, + 'headers', + {'Referer': '{}://{}'.format(redirectParsed.scheme, redirectParsed.netloc)} + ) + return self.request( + resp.request.method, + challengeSubmitResponse.headers['Location'], + **cloudflare_kwargs + ) # ------------------------------------------------------------------------------- # - # Cloudflare is doing http 3xx instead of pass through again.... + # We shouldn't be here... # Re-request the original query and/or process again.... # ------------------------------------------------------------------------------- # @@ -554,6 +587,17 @@ class CloudScraper(Session): # ------------------------------------------------------------------------------- # +if ssl.OPENSSL_VERSION_INFO < (1, 1, 1): + print( + "DEPRECATION: The OpenSSL being used by this python install ({}) does not meet the minimum supported " + "version (>= OpenSSL 1.1.1) in order to support TLS 1.3 required by Cloudflare, " + "You may encounter an unexpected reCaptcha or cloudflare 1020 blocks.".format( + ssl.OPENSSL_VERSION + ) + ) + +# ------------------------------------------------------------------------------- # + create_scraper = CloudScraper.create_scraper get_tokens = CloudScraper.get_tokens get_cookie_string = CloudScraper.get_cookie_string From 81248ed0bd13bb939a8d960adbb71e44dd840334 Mon Sep 17 00:00:00 2001 From: Alhaziel Date: Mon, 13 Jan 2020 17:40:09 +0100 Subject: [PATCH 3/4] Aggiornato AnimeWorld e CasaCinema --- channels.json | 8 +- channels/animeworld.py | 2 +- channels/casacinema.py | 229 ++++++++++++++++------------------------- 3 files changed, 93 insertions(+), 146 deletions(-) diff --git a/channels.json b/channels.json index 6e027a09..8347780e 100644 --- a/channels.json +++ b/channels.json @@ -2,18 +2,18 @@ "altadefinizione01": "https://www.altadefinizione01.tel", "altadefinizione01_link": "https://altadefinizione01.cam", "animeforce": "https://ww1.animeforce.org", - "animeleggendari": "https://animepertutti.com", + "animeleggendari": "https://animepertutti.com", "animestream": "https://www.animeworld.it", "animesubita": "http://www.animesubita.org", "animetubeita": "http://www.animetubeita.com", - "animeworld": "https://www1.animeworld.tv", - "casacinema": "https://www.casacinema.cloud", + "animeworld": "https://www.animeworld.cc", + "casacinema": "https://www.casacinema.biz", "casacinemaInfo": "https://casacinema.kim", "cb01anime": "https://www.cineblog01.ink", "cinemalibero": "https://www.cinemalibero.live", "cinetecadibologna": "http://cinestore.cinetecadibologna.it", "documentaristreamingda": "https://documentari-streaming-da.com", - "dreamsub": "https://www.dreamsub.stream", + "dreamsub": "https://dreamsub.stream", "fastsubita": "https://fastsubita.com", "filmgratis": "https://www.filmaltadefinizione.org", "filmigratis": "https://filmigratis.org", diff --git a/channels/animeworld.py b/channels/animeworld.py index 4fb56587..58a5b07a 100644 --- a/channels/animeworld.py +++ b/channels/animeworld.py @@ -10,7 +10,7 @@ headers = [['Referer', host]] __channel__ = 'animeworld' -list_servers = ['animeworld', 'verystream', 'streamango', 'openload', 'directo'] +list_servers = ['directo', 'animeworld', 'vvvvid'] list_quality = ['default', '480p', '720p', '1080p'] diff --git a/channels/casacinema.py b/channels/casacinema.py index 361aa152..083d5986 100644 --- a/channels/casacinema.py +++ b/channels/casacinema.py @@ -2,34 +2,12 @@ # ------------------------------------------------------------ # Canale per 'casacinema' # ------------------------------------------------------------ -""" - - Problemi noti che non superano il test del canale: - - Nella ricerca globale non sono presenti le voci: - - "Aggiungi in videoteca" - - "Scarica film/serie" - presenti però quando si entra nella pagina - - Avvisi: - Novità: - - Film, SerieTv - - Ulteriori info: - -""" -import re from core import support -from platformcode import config - -# in caso di necessità -from core import scrapertools, httptools -from core.item import Item -##### fine import -host = config.get_channel_url() +host = support.config.get_channel_url() headers = [['Referer', host]] list_servers = ['verystream', 'openload', 'wstream', 'speedvideo'] @@ -37,9 +15,6 @@ list_quality = ['HD', 'SD'] @support.menu def mainlist(item): - support.log(item) -## support.dbg() - film = ['/category/film', ('Generi', ['', 'genres', 'genres']), ('Sub-ITA', ['/category/sub-ita/', 'peliculas', 'sub']) @@ -53,24 +28,90 @@ def mainlist(item): return locals() + +@support.scrape +def genres(item): + action = 'peliculas' + blacklist = ['PRIME VISIONI', 'ULTIME SERIE TV', 'ULTIMI FILM'] + patronMenu = r'
  • (?P[^<>]+)</a></li>' + patronBlock = r'<div class="container home-cats">(?P<block>.*?)<div class="clear">' + return locals() + + +def select(item): + item.data = support.match(item)[1] + if 'continua con il video' in item.data.lower(): + support.log('select = ### è un film ###') + item.contentType = 'movie' + return findvideos(item) + else: + support.log('select = ### è una serie ###') + item.contentType = 'tvshow' + return episodios(item) + + +def search(item, text): + support.log(text) + text = text.replace(' ', '+') + item.url = host + '/?s=' + text + item.args = 'search' + try: + item.contentType = '' # non fa uscire le voci nel context menu + return peliculas(item) + + except: + import sys + for line in sys.exc_info(): + support.log('search log:', line) + return [] + + +def newest(categoria): + itemlist = [] + item = support.Item() + item.args = 'newest' + + try: + if categoria == 'series': + item.contentType = 'tvshow' + item.url = host+'/aggiornamenti-serie-tv' + + else: + item.contentType = 'movie' + item.url = host+'/category/film' + + item.action = 'peliculas' + itemlist = peliculas(item) + + if itemlist[-1].action == 'peliculas': + itemlist.pop() + + # Continua la ricerca in caso di errore + except: + import sys + for line in sys.exc_info(): + support.log('newest log: ', {0}.format(line)) + return [] + + return itemlist + + @support.scrape def peliculas(item): - support.log(item) -## support.dbg() # decommentare per attivare web_pdb - if item.contentType == 'movie': action = 'findvideos' elif item.contentType == 'tvshow': action = 'episodios' pagination = '' else: - # è una ricerca action = 'select' - blacklist = [''] - patron = r'<li><a href="(?P<url>[^"]+)"[^=]+="(?P<thumb>[^"]+)"><div> <div[^>]+>(?P<title>.*?)[ ]?(?:\[(?P<quality1>HD)\])?[ ]?(?:\(|\[)?(?P<lang>Sub-ITA)?(?:\)|\])?[ ]?(?:\[(?P<quality>.+?)\])?[ ]?(?:\((?P<year>\d+)\))?<(?:[^>]+>.+?(?:title="Nuovi episodi">(?P<episode>\d+x\d+)[ ]?(?P<lang2>Sub-Ita)?|title="IMDb">(?P<rating>[^<]+)))?' - patronBlock = r'<h1>.+?</h1>(?P<block>.*?)<aside>' - patronNext = '<a href="([^"]+)" >Pagina' + if item.args == 'newest': + patron = r'<li><a href="(?P<url>[^"]+)"[^=]+="(?P<thumb>[^"]+)"><div> <div[^>]+>(?P<title>[^\(\[<]+)(?:\[(?P<quality1>HD)\])?[ ]?(?:\(|\[)?(?P<lang>Sub-ITA)?(?:\)|\])?[ ]?(?:\[(?P<quality>.+?)\])?[ ]?(?:\((?P<year>\d+)\))?<(?:[^>]+>.+?(?:title="Nuovi episodi">(?P<episode>\d+x\d+)[ ]?(?P<lang2>Sub-Ita)?|title="IMDb">(?P<rating>[^<]+)))?' + else: + patron = r'<li><a href="(?P<url>[^"]+)"[^=]+="(?P<thumb>[^"]+)"><div> <div[^>]+>(?P<title>[^\(\[<]+)(?:\[(?P<quality1>HD)\])?[ ]?(?:\(|\[)?(?P<lang>Sub-ITA)?(?:\)|\])?[ ]?(?:\[(?P<quality>.+?)\])?[ ]?(?:\((?P<year>\d+)\))?<' + + patronNext = r'<a href="([^"]+)" >Pagina' def itemHook(item): if item.quality1: @@ -82,125 +123,31 @@ def peliculas(item): if item.args == 'novita': item.title = item.title return item - -## debug = True # True per testare le regex sul sito return locals() + @support.scrape def episodios(item): - support.log(item) - #dbg - if item.data1: - data = item.data1 + if item.data: + data = item.data action = 'findvideos' item.contentType = 'tvshow' blacklist = [''] patron = r'(?P<episode>\d+(?:×|×)?\d+\-\d+|\d+(?:×|×)\d+)[;]?(?:(?P<title>[^<]+)<(?P<url>.*?)|(\2[ ])(?:<(\3.*?)))(?:<br />|</p>)' patronBlock = r'<strong>(?P<block>(?:.+?Stagione*.+?(?P<lang>[Ii][Tt][Aa]|[Ss][Uu][Bb][\-]?[iI][tT][aA]))?(?:.+?|</strong>)(/?:</span>)?</p>.*?</p>)' - -## debug = True return locals() -# Questa def è utilizzata per generare il menu 'Generi' del canale -# per genere, per anno, per lettera, per qualità ecc ecc -@support.scrape -def genres(item): - support.log(item) - #dbg - - action = 'peliculas' - blacklist = ['PRIME VISIONI', 'ULTIME SERIE TV', 'ULTIMI FILM'] - patron = r'<li><a href="(?P<url>[^"]+)">(?P<title>[^<>]+)</a></li>' - patronBlock = r'<div class="container home-cats">(?P<block>.*?)<div class="clear">' - - #debug = True - return locals() - -def select(item): - support.log('select --->', item) -## debug = True - #support.dbg() - data = httptools.downloadpage(item.url, headers=headers).data - data = re.sub('\n|\t', ' ', data) - data = re.sub(r'>\s+<', '> <', data) - if 'continua con il video' in data.lower(): -## block = scrapertools.find_single_match(data, r'<div class="col-md-8 bg-white rounded-left p-5"><div>(.*?)<div style="margin-left: 0.5%; color: #FFF;">') -## if re.findall('rel="category tag">serie', data, re.IGNORECASE): - support.log('select = ### è un film ###') - return findvideos(Item(channel=item.channel, - title=item.title, - fulltitle=item.fulltitle, - url=item.url, - #args='serie', - contentType='movie', - data1 = data - )) - else: - support.log('select = ### è una serie ###') - return episodios(Item(channel=item.channel, - title=item.title, - fulltitle=item.fulltitle, - url=item.url, - #args='serie', - contentType='tvshow', - data1 = data - )) - -############## Fondo Pagina - -def search(item, text): - support.log('search ->', item) - itemlist = [] - text = text.replace(' ', '+') - item.url = host + '/?s=' + text - item.args = 'search' - try: - item.contentType = 'episode' # non fa uscire le voci nel context menu - return peliculas(item) - # Se captura la excepcion, para no interrumpir al buscador global si un canal falla - except: - import sys - for line in sys.exc_info(): - support.log('search log:', line) - return [] - -def newest(categoria): - support.log('newest ->', categoria) - itemlist = [] - item = Item() - - try: - if categoria == 'series': - item.contentType = 'tvshow' - item.url = host+'/aggiornamenti-serie-tv' - item.args = 'novita' - else: - item.contentType = 'movie' - item.url = host+'/category/film' - - item.action = 'peliculas' - itemlist = peliculas(item) - - if itemlist[-1].action == 'peliculas': - itemlist.pop() - # Continua la ricerca in caso di errore - except: - import sys - for line in sys.exc_info(): - support.log('newest log: ', {0}.format(line)) - return [] - - return itemlist def findvideos(item): - support.log('findvideos ->', item) - itemlist = [] if item.contentType != 'movie': - return support.server(item, item.url) + links = support.match(item.url, r'href="([^"]+)"')[0] else: - links = str(support.match(item, r'SRC="([^"]+)"', patronBlock=r'<div class="col-md-10">(.+?)<div class="swappable" id="links">')[0]) - if links: - links = links.replace('#', 'speedvideo.net') - return support.server(item, links) - else: - return support.server(item) + matchData = item.data if item.data else item + links = support.match(matchData, r'(?:SRC|href)="([^"]+)"', patronBlock=r'<div class="col-md-10">(.+?)<div class="ads">')[0] + data = '' + from lib.unshortenit import unshorten_only + for link in links: + support.log('URL=',link) + url, c = unshorten_only(link.replace('#', 'speedvideo.net')) + data += url + '\n' + return support.server(item, data) From 13c00d5655bfa9534225db6629ccd4946c52a4bb Mon Sep 17 00:00:00 2001 From: Alhaziel <alhaziel01@gmail.com> Date: Mon, 13 Jan 2020 19:24:25 +0100 Subject: [PATCH 4/4] Riscritto DreamSub --- channels/dreamsub.json | 1 - channels/dreamsub.py | 320 +++++++++++------------------------------ core/httptools.py | 2 +- 3 files changed, 83 insertions(+), 240 deletions(-) diff --git a/channels/dreamsub.json b/channels/dreamsub.json index ba8bf611..d0d39ea9 100644 --- a/channels/dreamsub.json +++ b/channels/dreamsub.json @@ -7,6 +7,5 @@ "thumbnail": "dreamsub.png", "banner": "dreamsub.png", "categories": ["anime", "vos"], - "not_active": ["include_in_newest"], "settings": [] } diff --git a/channels/dreamsub.py b/channels/dreamsub.py index e5ac914c..4415224e 100644 --- a/channels/dreamsub.py +++ b/channels/dreamsub.py @@ -2,214 +2,53 @@ # ------------------------------------------------------------ # Canale per 'dreamsub' # ------------------------------------------------------------ -# ------------------------------------------------------------ -""" - - Problemi noti che non superano il test del canale: - - Nessuno noto! - - Avvisi per i tester: - 1. Gli episodi sono divisi per pagine di 20 - 2. In Novità->Anime, cliccare sulla home il bottone "Ultime inserite" - Se avete più titoli in KOD, ridimensiona il browser in modo che si vedano i titoli - a gruppi di 3 e ricontrollare, è un problema del sito. - - 3.Passaggi per Aggiungere in videoteca e/o scaricare Serie: - 1. sul titolo -> menu contestuale -> Rinumerazione - Solo dopo questo passaggio appariranno le voci, sul titolo -> menu contestuale ->: - - Aggiungi in videoteca (senza rinumerazione non appare - la voce) - - Scarica Serie e Scarica Stagione ( Se download Abilitato! ) - - 4. ### PIù IMPORTANTE!!! ### - #### NON E' DA CONSIDERARE ERRORE NEL TEST QUANTO RIPORTATO DI SEGUITO!!!! #### - 1. Il sito permette un filtro tra anime e film, tramite url. - Se nell'url c'è /anime/, sul titolo e proseguendo fino alla pagina del video, saranno - presenti le voci: - - 'Rinumerazione', prima, e dopo: 'Aggiungi in videoteca', 'Scarica Serie' etc... - Tutto il resto è trattato come film e si avranno le voci solite: - AD eccezione per quei "FILM" che hanno 2 o più titoli all'interno, in questo caso: - 1. Non apparirà nessuna voce tra "Aggiungi in videoteca" e "Scarica Film" e nemmeno "rinumerazione" - 2. Dopo essere entrato nella pagina del Titolo Principale, troverai una lista di titoli dove sarà possibile scaricare - il filmato (chiamato EPISODIO) stessa cosa accedendo alla pagina ultima del video - 3. Questi TITOLI NON POSSONO ESSERE AGGIUNTI IN VIDEOTECA - le voci "Scarica FILM" si avranno dopo. - - Es: - https://www.dreamsub.stream/movie/5-centimetri-al-secondo -> film ma ha 3 titoli - - Il Canale NON è presente nelle novità(globale) -> Anime - - -""" -# Qui gli import -import re from core import support -from platformcode import config -from core import scrapertools, httptools, servertools, tmdb -from core.item import Item -##### fine import -host = config.get_channel_url() +host = support.config.get_channel_url() headers = [['Referer', host]] -# server di esempio... -list_servers = ['directo', 'verystream', 'streamango', 'openload'] -# quality di esempio -list_quality = ['default'] - -#### Inizio delle def principali ### - @support.menu def mainlist(item): support.log(item) - anime = ['/anime', -## ('Novità', ['']), -## ('OAV', ['/search/oav', 'peliculas', 'oav']), -## ('OVA', ['/search/ova', 'peliculas', 'ova']), - ('Movie', ['/search/movie', 'peliculas', '', 'movie']), - ('Film', ['/search/film', 'peliculas', '', 'movie']), - ('Categorie', ['/filter?genere=','genres']), -## ('Ultimi Episodi', ['', 'last']) + anime = ['/search?typeY=tv', + ('Movie', ['/search?typeY=movie', 'peliculas', '', 'movie']), + ('OAV', ['/search?typeY=oav', 'peliculas', '', 'tvshow']), + ('Spinoff', ['/search?typeY=spinoff', 'peliculas', '', 'tvshow']), + ('Generi', ['','menu','Generi']), + ('Stato', ['','menu','Stato']), + ('Ultimi Episodi', ['', 'peliculas', ['last', 'episodiRecenti']]), + ('Ultimi Aggiornamenti', ['', 'peliculas', ['last', 'episodiNuovi']]) ] return locals() -@support.scrape -def peliculas(item): - support.log(item) - #dbg # decommentare per attivare web_pdb - - anime = True - if item.args == 'newest': - patronBlock = r'<div class="showRoomGoLeft" sr="ultime"></div>(?P<block>.*?)<div class="showRoomGoRight" sr="ultime">' - else: - patronBlock = r'<input type="submit" value="Vai!" class="blueButton">(?P<block>.*?)<div class="footer">' - -## patron = r'<div class="showStreaming"> <b>(?P<title>[^<]+).+?Stato streaming: '\ -## '(?:[^<]+)<.*?Lingua:[ ](?P<lang1>ITA\/JAP|ITA|JAP)?(?:[ ])?'\ -## '(?P<lang2>SUB ITA)?<br>.+?href="(?P<url>[^"]+)".+?'\ -## 'background: url\((?P<thumb>[^"]+)\).+?<div class="tvTitle">.+?'\ -## '<strong>Anno di inizio</strong>: (?P<year>\d+)<br>' - - patron = r'<div class="showStreaming"> <b>(?P<title>[^<]+).+?Stato streaming: (?:[^<]+)<.*?Lingua:[ ](?P<lang1>ITA\/JAP|ITA|JAP)?(?:[ ])?(?P<lang2>SUB ITA)?<br>.+?href="(?P<url>[^"]+)".+?background: url\((?P<thumb>[^"]+)\).+?<div class="tvTitle">.+?Episodi[^>]+>.\s?(?P<nep>\d+).+?<strong>Anno di inizio</strong>: (?P<year>\d+)<br>' - patronNext = '<li class="currentPage">[^>]+><li[^<]+<a href="([^"]+)">' - - def itemHook(item): - support.log("ITEMHOOK -> ", item) - item = language(item) - - if 'anime' in item.url: - item.contentType = 'tvshow' - item.action = 'episodios' - #item.args = 'anime' - else: - if item.nep == '1': - item.contentType = 'movie' - item.action = 'findvideos' - else: - item.contentType = 'episode' - item.args = '' - item.nep = item.nep - item.action = 'findmovie' - return item - - #debug = True - return locals() @support.scrape -def episodios(item): - support.log(item) - #support.dbg() - - action = 'findvideos' - patronBlock = r'<div class="seasonEp">(?P<block>.*?)<div class="footer">' - patron = r'<li><a href="(?P<url>[^"]+)"[^<]+<b>(?:.+?)[ ](?P<episode>\d+)<\/b>[^>]+>(?P<title>[^<]+)<\/i>[ ]\(?(?P<lang1>ITA|Sub ITA)?\s?.?\s?(?P<lang2>Sub ITA)?.+?\)?<\/a>' - - def itemHook(item): - item = language(item) - return item - - pagination = '' - - #debug = True - return locals() - -@support.scrape -def genres(item): - support.log(item) - #dbg - +def menu(item): item.contentType = '' action = 'peliculas' - blacklist = ['tutti'] - patron = r'<option value="(?P<title>[^"]+)">' - patronBlock = r'<select name="genere" id="genere" class="selectInput">(?P<block>.*?)</select>' + + patronBlock = r'<div class="filter-header"><b>%s</b>(?P<block>.*?)<div class="filter-box">' % item.args + patronMenu = r'<a class="[^"]+" data-state="[^"]+" (?P<url>[^>]+)>[^>]+></i>[^>]+></i>[^>]+></i>(?P<title>[^>]+)</a>' def itemHook(item): - item.contentTitle = item.contentTitle.replace(' ', '+') - item.url = host+'/filter?genere='+item.contentTitle + for Type, ID in support.match(item.url, r'data-type="([^"]+)" data-id="([^"]+)"')[0]: + item.url = host + '/search?' + Type + 'Y=' + ID return item - - #debug = True return locals() -@support.scrape -def findmovie(item): - support.log(item) - - patronBlock = r'<div class="seasonEp">(?P<block>.*?)<div class="footer">' - item.contentType = 'episode' - item.nep = 2 - patron = r'<li><a href="(?P<url>[^"]+)"[^>]+>.(?P<title2>.+?)-.+?-[ ]<b>(?P<title>.+?)</b>\s+\(?(?P<lang1>ITA)?\s?(?P<lang2>Sub ITA)?.+?\)?' - - def itemHook(item): - item = language(item) - return item - - #debug = True - return locals() - - -def language(item): - lang = [] - - if item.lang1: - if item.lang1.lower() == 'ita/jap' or item.lang1.lower() == 'ita': - lang.append('ITA') - - if item.lang1.lower() == 'jap' and item.lang1.lower() == 'sub ita': - lang.append('Sub-ITA') - - if item.lang2: - if item.lang2.lower() == 'sub ita': - lang.append('Sub-ITA') - - item.contentLanguage = lang - - if len(lang) ==2: - item.title += support.typo(lang[0], '_ [] color kod') + support.typo(lang[1], '_ [] color kod') - #item.show += support.typo(lang[0], '_ [] color kod') + support.typo(lang[1], '_ [] color kod') - elif len(lang) == 1: - item.title += support.typo(lang[0], '_ [] color kod') - #item.show += support.typo(lang[0], '_ [] color kod') - - return item - - def search(item, text): - support.log('search', item) - itemlist = [] + support.log(text) text = text.replace(' ', '+') item.url = host + '/search/' + text item.args = 'search' try: return peliculas(item) - # Se captura la excepcion, para no interrumpir al buscador global si un canal falla + # Continua la ricerca in caso di errore except: import sys for line in sys.exc_info(): @@ -217,77 +56,82 @@ def search(item, text): return [] -# da adattare... ( support.server ha vari parametri ) -#support.server(item, data='', itemlist=[], headers='', AutoPlay=True, CheckLinks=True) +def newest(categoria): + support.log(categoria) + item = support.Item() + try: + if categoria == "anime": + item.url = host + item.args = ['last', 'episodiNuovi'] + return peliculas(item) + # Continua la ricerca in caso di errore + except: + import sys + for line in sys.exc_info(): + support.logger.error("{0}".format(line)) + return [] + + + +@support.scrape +def peliculas(item): + anime = True + if 'movie' in item.url: + item.contentType = 'movie' + action = 'findvideos' + else: + item.contentType = 'tvshow' + action = 'episodios' + + if len(item.args) > 1 and item.args[0] == 'last': + patronBlock = r'<div id="%s"[^>]+>(?P<block>.*?)<div class="vistaDettagliata"' % item.args[1] + patron = r'<li>\s*<a href="(?P<url>[^"]+)" title="(?P<title>[^"]+)" class="thumb">[^>]+>[^>]+>[^>]+>\s*[EePp]+\s*(?P<episode>\d+)[^>]+>[^>]+>[^>]+>(?P<lang>[^<]*)<[^>]+>[^>]+>\s<img src="(?P<thumb>[^"]+)"' + else: + patron = r'<div class="showStreaming"> <b>(?P<title>[^<]+)[^>]+>[^>]+>\s*Stato streaming: (?:[^<]+)<[^>]+>[^>]+>\s*Lingua:[ ](?P<lang>ITA\/JAP|ITA|JAP|SUB ITA)?[^>]+>[^>]+>\s*<a href="(?P<url>[^"]+)"[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>[^>]+>\s*<div class="[^"]+" style="background: url\((?P<thumb>[^\)]+)\)' + patronNext = '<li class="currentPage">[^>]+><li[^<]+<a href="([^"]+)">' + + return locals() + + +@support.scrape +def episodios(item): + anime = True + pagination = 100 + + if item.data: + data = item.data + + patron = r'<div class="sli-name">\s*<a href="(?P<url>[^"]+)"[^>]+>(?P<title>[^<]+)<' + + return locals() + + def findvideos(item): - support.log("ITEM ---->", item) itemlist = [] + support.log() - data = httptools.downloadpage(item.url).data - data = re.sub(r'\n|\t', ' ', data) - data = re.sub(r'>\s\s*<', '><', data) - patronBlock = r'LINK STREAMING(?P<block>.*?)LINK DOWNLOAD' - patron = r'href="(.+?)"' - block = scrapertools.find_single_match(data, patronBlock) - urls = scrapertools.find_multiple_matches(block, patron) - #support.regexDbg(item, patron, headers, data=data) + matches, data = support.match(item, r'<a href="([^"]+)"', r'<div style="white-space: (.*?)<div id="main-content"') - for url in urls: - titles = item.infoLabels['title'] - lang = '' - if 'sub_ita' in url.lower(): - lang = 'Sub-ITA' - else: - lang = 'ITA' + if not matches: + item.data = data + item.contentType = 'tvshow' + return episodios(item) - if 'keepem.online' in data: - urls = scrapertools.find_multiple_matches(data, r'(https://keepem\.online/f/[^"]+)"') - for url in urls: - url = httptools.downloadpage(url).url - itemlist += servertools.find_video_items(data=url) + matches.sort() - elif 'keepsetsu' in url.lower() or 'woof' in url.lower(): - if 'keepsetsu' in url.lower(): - support.log("keepsetsu url -> ", url ) - data = httptools.downloadpage(url).url - support.log("LINK-DATA :", data) - - data = httptools.downloadpage(data).data - support.log("LINK-DATA2 :", data) - video_urls = scrapertools.find_single_match(data, r'<meta name="description" content="([^"]+)"') - - else: - - data = httptools.downloadpage(url).data - #host_video = scrapertools.find_single_match(data, r'var thisPageUrl = "(http[s]\:\/\/[^\/]+).+?"') - host_video = scrapertools.find_single_match(data, r'(?:let|var) thisPageUrl = "(http[s]\:\/\/[^\/]+).+?"') - link = scrapertools.find_single_match(data, r'<video src="([^"]+)"') - video_urls = host_video+link - - title_show = support.typo(titles,'_ bold') + support.typo(lang,'_ [] color kod') + for url in matches: + lang = url.split('/')[-2] + quality = url.split('/')[-1] itemlist.append( support.Item(channel=item.channel, action="play", contentType=item.contentType, - title=title_show, - fulltitle=item.fulltitle, - show=item.fulltitle, - url=link if 'http' in link else video_urls, - infoLabels = item.infoLabels, - thumbnail=item.thumbnail, - contentSerieName= item.fulltitle, - contentTitle=title_show, - contentLanguage = 'ITA' if lang == [] else lang, - args=item.args, + title=lang, + url=url, + contentLanguage = lang, + quality = quality, server='directo', )) - if item.contentType != 'episode' and int(item.nep) < 2 : - # Link Aggiungi alla Libreria - if config.get_videolibrary_support() and len(itemlist) > 0 and item.extra != 'findservers': - support.videolibrary(itemlist, item) - # link per scaricare - if config.get_setting('downloadenabled'): - support.download(itemlist, item) - return itemlist + return support.server(item, itemlist=itemlist) \ No newline at end of file diff --git a/core/httptools.py b/core/httptools.py index 526b555e..505828d3 100755 --- a/core/httptools.py +++ b/core/httptools.py @@ -253,7 +253,7 @@ def downloadpage(url, **opt): load_cookies() domain = urlparse.urlparse(url).netloc CF = False - if domain in ['www.guardaserie.media', 'casacinema.space']: + if domain in ['www.guardaserie.media', 'casacinema.space', 'dreamsub.stream']: from lib import cloudscraper session = cloudscraper.create_scraper() CF = True