From 0692132a3da80b50bcd971ee758657563dd7f0e2 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 27 Dec 2019 23:03:21 +0100 Subject: [PATCH] pulizia --- channels/0example.py.txt | 6 +- channels/altadefinizione01.py | 4 +- channels/animeforce.py | 4 +- channels/animetubeita.py | 4 +- channels/animeworld.py | 6 +- channels/casacinema.py | 4 +- channels/casacinemaInfo.py | 4 +- channels/cineblog01.py | 10 +- channels/cinemalibero.py | 8 +- channels/dreamsub.py | 16 +- channels/eurostreaming.py | 6 +- channels/fastsubita.py | 4 +- channels/filmpertutti.py | 4 +- channels/hdblog.json | 37 -- channels/hdblog.py | 94 --- channels/ilgeniodellostreaming.py | 6 +- channels/ilgiramondo.json | 37 -- channels/ilgiramondo.py | 67 -- channels/istitutoluce.json | 37 -- channels/istitutoluce.py | 288 --------- channels/italiaserie.py | 6 +- channels/programmazione.json | 21 - channels/programmazione.py | 71 -- channels/renumbertools.py | 1001 ----------------------------- channels/ricettevideo.json | 37 -- channels/ricettevideo.py | 58 -- channels/seriehd.py | 4 +- channels/serietvonline.py | 8 +- channels/serietvu.py | 6 +- channels/streamtime.py | 6 +- channels/tantifilm.py | 28 +- channels/vedohd.py | 4 +- core/cloudflare.py | 112 ---- core/httptools.py | 14 +- core/scrapertools.py | 158 +++-- core/scrapertoolsV2.py | 346 ---------- core/servertools.py | 4 +- core/support.py | 62 +- platformcode/platformtools.py | 4 +- servers/decrypters/zcrypt.py | 6 +- servers/hdload.py | 2 +- servers/mixdrop.py | 8 +- servers/onlystream.py | 8 +- servers/supervideo.py | 8 +- specials/autorenumber.py | 16 +- specials/community.py | 4 +- specials/searchall.py | 866 ------------------------- 47 files changed, 218 insertions(+), 3296 deletions(-) delete mode 100644 channels/hdblog.json delete mode 100644 channels/hdblog.py delete mode 100644 channels/ilgiramondo.json delete mode 100644 channels/ilgiramondo.py delete mode 100644 channels/istitutoluce.json delete mode 100644 channels/istitutoluce.py delete mode 100644 channels/programmazione.json delete mode 100644 channels/programmazione.py delete mode 100644 channels/renumbertools.py delete mode 100644 channels/ricettevideo.json delete mode 100644 channels/ricettevideo.py delete mode 100644 core/cloudflare.py delete mode 100644 core/scrapertoolsV2.py delete mode 100644 specials/searchall.py diff --git a/channels/0example.py.txt b/channels/0example.py.txt index 40f8d6ed..2d5bb5ea 100644 --- a/channels/0example.py.txt +++ b/channels/0example.py.txt @@ -48,7 +48,7 @@ from core import support from platformcode import config # in caso di necessità -#from core import scrapertoolsV2, httptools, servertools, tmdb +#from core import scrapertools, httptools, servertools, tmdb from core.item import Item # per newest #from lib import unshortenit @@ -74,7 +74,7 @@ def findhost(): permUrl = httptools.downloadpage('INSERIRE-URL-QUI', follow_redirects=False).headers host = 'https://www.'+permUrl['location'].replace('https://www.google.it/search?q=site:', '') # cancellare host non utilizzato - host = scrapertoolsV2.find_single_match(permUrl, r'
\s+<', '> <', data) - block = scrapertoolsV2.find_single_match(data, r'') + block = scrapertools.find_single_match(data, r'') if re.findall('', data, re.IGNORECASE): support.log('select = ### è una serie ###') return episodios(Item(channel=item.channel, diff --git a/channels/altadefinizione01.py b/channels/altadefinizione01.py index cf1764df..afc6e518 100644 --- a/channels/altadefinizione01.py +++ b/channels/altadefinizione01.py @@ -13,7 +13,7 @@ Ulteriori info: """ -from core import scrapertoolsV2, httptools, support +from core import scrapertools, httptools, support from core.item import Item from platformcode import config, logger @@ -21,7 +21,7 @@ from platformcode import config, logger def findhost(): data = httptools.downloadpage('https://altadefinizione01-nuovo.link/').data - host = scrapertoolsV2.find_single_match(data, '
""") - if not url: url = support.scrapertoolsV2.find_single_match(data, 'file: "([^"]+)"') + url = support.scrapertools.find_single_match(data, """""") + if not url: url = support.scrapertools.find_single_match(data, 'file: "([^"]+)"') if url: url += '|' + urllib.urlencode(headers) itemlist.append( diff --git a/channels/animeworld.py b/channels/animeworld.py index 41bc678e..3b50b070 100644 --- a/channels/animeworld.py +++ b/channels/animeworld.py @@ -145,9 +145,9 @@ def findvideos(item): videoData = '' for serverid in matches: - if not item.number: item.number = support.scrapertoolsV2.find_single_match(item.title,r'(\d+) -') - block = support.scrapertoolsV2.find_multiple_matches(data,'data-id="' + serverid + '">(.*?)
(.*?)
\s+<', '> <', data) if 'continua con il video' in data.lower(): -## block = scrapertoolsV2.find_single_match(data, r'
(.*?)
') +## block = scrapertools.find_single_match(data, r'
(.*?)
') ## if re.findall('rel="category tag">serie', data, re.IGNORECASE): support.log('select = ### è un film ###') return findvideos(Item(channel=item.channel, diff --git a/channels/casacinemaInfo.py b/channels/casacinemaInfo.py index 1be00a6b..b924139a 100644 --- a/channels/casacinemaInfo.py +++ b/channels/casacinemaInfo.py @@ -19,13 +19,13 @@ """ from core import support -from core import scrapertoolsV2, httptools +from core import scrapertools, httptools from core.item import Item def findhost(): data = httptools.downloadpage('https://casacinema.nuovo.link').data - host = scrapertoolsV2.find_single_match(data, r'
]+>([^<]+)<' @@ -207,7 +207,7 @@ def findvideos(item): matches = re.compile(patronvideos, re.DOTALL).finditer(data) QualityStr = "" for match in matches: - QualityStr = scrapertoolsV2.decodeHtmlentities(match.group(1))[6:] + QualityStr = scrapertools.decodeHtmlentities(match.group(1))[6:] # Estrae i contenuti - Streaming load_links(itemlist, 'Streamin?g:(.*?)cbtable', "orange", "Streaming", "SD") @@ -307,12 +307,12 @@ def play(item): data = httptools.downloadpage(item.url).data if "window.location.href" in data: try: - data = scrapertoolsV2.find_single_match(data, 'window.location.href = "([^"]+)";') + data = scrapertools.find_single_match(data, 'window.location.href = "([^"]+)";') except IndexError: data = httptools.downloadpage(item.url, only_headers=True, follow_redirects=False).headers.get("location", "") data, c = unshortenit.unwrap_30x_only(data) else: - data = scrapertoolsV2.find_single_match(data, r'.*?licca.*?') + data = scrapertools.find_single_match(data, r'.*?licca.*?') logger.debug("##### play go.php data ##\n%s\n##" % data) else: diff --git a/channels/cinemalibero.py b/channels/cinemalibero.py index d39fc1f8..0aee93a2 100644 --- a/channels/cinemalibero.py +++ b/channels/cinemalibero.py @@ -5,7 +5,7 @@ import re -from core import httptools, support, scrapertoolsV2 +from core import httptools, support, scrapertools from core.item import Item from platformcode import config @@ -89,7 +89,7 @@ def episodios(item): patronBlock = r'

(?:.+?[Ss]tagione\s)?(?:(?PiTA|ITA|Sub-ITA|Sub-iTA))?.*?(?P.+?)(?:(.+?

(.+?
", data) if 'clicca qui per aprire' in data.lower(): - url = scrapertoolsV2.find_single_match(data, '"go_to":"([^"]+)"') + url = scrapertools.find_single_match(data, '"go_to":"([^"]+)"') url = url.replace("\\","") # Carica la pagina data = httptools.downloadpage(url, headers=headers).data.replace("'", '"') elif 'clicca qui' in data.lower(): - url = scrapertoolsV2.find_single_match(data, '

') + url = scrapertools.find_single_match(data, '

') # Carica la pagina data = httptools.downloadpage(url, headers=headers).data.replace("'", '"') diff --git a/channels/fastsubita.py b/channels/fastsubita.py index 6acad4aa..b022c94e 100644 --- a/channels/fastsubita.py +++ b/channels/fastsubita.py @@ -16,7 +16,7 @@ - SOLO SUB-ITA """ -from core import support, httptools, scrapertoolsV2 +from core import support, httptools, scrapertools from core.item import Item from core.support import log from platformcode import config @@ -158,7 +158,7 @@ def findvideos(item): data = httptools.downloadpage(item.url).data patron = r'>Posted in (.*?)') + patronBlock = scrapertools.find_single_match(data, r'class="taxonomy category" >(.*?)') if patronBlock.lower() != 'film': support.log('select = ### è una serie ###') item.contentType='tvshow' diff --git a/channels/hdblog.json b/channels/hdblog.json deleted file mode 100644 index a352408c..00000000 --- a/channels/hdblog.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "id": "hdblog", - "name": "Hdblog", - "language": ["ita"], - "active": true, - "adult": false, - "thumbnail": "http://css.hd-cdn.it/new_files/templates/theme_darklight/img/logos_wt/logohdhardware.png", - "banner": "http://css.hd-cdn.it/new_files/templates/theme_darklight/img/logos_wt/logohdhardware.png", - "categories": ["documentary"], - "settings": [ - { - "id": "include_in_global_search", - "type": "bool", - "label": "Includi in Ricerca Globale", - "default": false, - "enabled": false, - "visible": false - }, - { - "id": "include_in_newest_documentales", - "type": "bool", - "label": "Includi in Novità - Documentari", - "default": true, - "enabled": true, - "visible": true - }, - { - "id": "include_in_newest_italiano", - "type": "bool", - "label": "Includi in Novità - Italiano", - "default": true, - "enabled": true, - "visible": true - } - ] -} - diff --git a/channels/hdblog.py b/channels/hdblog.py deleted file mode 100644 index c52e4266..00000000 --- a/channels/hdblog.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------ -# Ringraziamo Icarus crew -# Canale hdblog -# ------------------------------------------------------------ -import re -import urlparse - -from core import httptools, scrapertools -from core.item import Item -from platformcode import logger -from platformcode import config - -host = "https://www.hdblog.it" - - -def mainlist(item): - logger.info("kod.hdblog mainlist") - itemlist = [Item(channel=item.channel, - title="[COLOR azure]Video recensioni tecnologiche[/COLOR]", - action="peliculas", - url=host + "/video/", - thumbnail="http://www.crat-arct.org/uploads/images/tic%201.jpg"), - Item(channel=item.channel, - title="[COLOR azure]Categorie[/COLOR]", - action="categorias", - url=host + "/video/", - thumbnail="http://www.crat-arct.org/uploads/images/tic%201.jpg")] - - return itemlist - - -def categorias(item): - logger.info("kod.hdblog categorias") - itemlist = [] - - data = httptools.downloadpage(item.url).data - logger.info(data) - - # Narrow search by selecting only the combo - start = data.find('
') - end = data.find('
', start) - bloque = data[start:end] - - # The categories are the options for the combo - patron = ']+>(.*?)' - matches = re.compile(patron, re.DOTALL).findall(bloque) - scrapertools.printMatches(matches) - - for scrapedurl, scrapedtitle in matches: - scrapedthumbnail = "" - scrapedplot = "" - itemlist.append( - Item(channel=item.channel, - action="peliculas", - title="[COLOR azure]" + scrapedtitle + "[/COLOR]", - url=scrapedurl + "video/", - thumbnail=scrapedthumbnail, - plot=scrapedplot)) - - return itemlist - - -def peliculas(item): - logger.info("kod.hdblog peliculas") - itemlist = [] - - # Carica la pagina - data = httptools.downloadpage(item.url).data - - # Estrae i contenuti - patron = '\s*]+>\s*\s*[^>]+>\s*(.*?)\s*<' - matches = re.compile(patron, re.DOTALL).findall(data) - scrapertools.printMatches(matches) - - for scrapedurl, scrapedthumbnail, scrapedtitle in matches: - scrapedtitle = scrapertools.decodeHtmlentities(scrapedtitle) - scrapedplot = "" - itemlist.append(Item(channel=item.channel, action="findvideos", fulltitle=scrapedtitle, show=scrapedtitle, - title=scrapedtitle, url=scrapedurl, thumbnail=scrapedthumbnail, plot=scrapedplot, - folder=True)) - - # Paginazione - patronvideos = '[^>]+>[^=]+="next" href="(.*?)" class="inattiva">' - matches = re.compile(patronvideos, re.DOTALL).findall(data) - scrapertools.printMatches(matches) - - if len(matches) > 0: - scrapedurl = urlparse.urljoin(item.url, matches[0]) - itemlist.append( - Item(channel=item.channel, action="peliculas", title="[COLOR orange]Avanti >>[/COLOR]", url=scrapedurl, - folder=True)) - - return itemlist diff --git a/channels/ilgeniodellostreaming.py b/channels/ilgeniodellostreaming.py index 10defaf6..3cbb560e 100644 --- a/channels/ilgeniodellostreaming.py +++ b/channels/ilgeniodellostreaming.py @@ -33,7 +33,7 @@ import re -from core import scrapertoolsV2, httptools, support +from core import scrapertools, httptools, support from core.support import log from core.item import Item from platformcode import config @@ -216,7 +216,7 @@ def findvideos(item): matches, data = support.match(item, '", "") - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("

", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("

", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("", re.DOTALL).sub("", cadena) - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - cadena = cadena.replace("", "") - - cadena = re.compile("]*>", re.DOTALL).sub("", cadena) - - cadena = cadena.replace("\t", "") - # cadena = entityunescape(cadena) - return cadena - - -def slugify(title): - # print title - - # Sustituye acentos y eñes - title = title.replace("Á", "a") - title = title.replace("É", "e") - title = title.replace("Í", "i") - title = title.replace("Ó", "o") - title = title.replace("Ú", "u") - title = title.replace("á", "a") - title = title.replace("é", "e") - title = title.replace("í", "i") - title = title.replace("ó", "o") - title = title.replace("ú", "u") - title = title.replace("À", "a") - title = title.replace("È", "e") - title = title.replace("Ì", "i") - title = title.replace("Ò", "o") - title = title.replace("Ù", "u") - title = title.replace("à", "a") - title = title.replace("è", "e") - title = title.replace("ì", "i") - title = title.replace("ò", "o") - title = title.replace("ù", "u") - title = title.replace("ç", "c") - title = title.replace("Ç", "C") - title = title.replace("Ñ", "n") - title = title.replace("ñ", "n") - title = title.replace("/", "-") - title = title.replace("&", "&") - - # Pasa a minúsculas - title = title.lower().strip() - - # Elimina caracteres no válidos - validchars = "abcdefghijklmnopqrstuvwxyz1234567890- " - title = ''.join(c for c in title if c in validchars) - - # Sustituye espacios en blanco duplicados y saltos de línea - title = re.compile("\s+", re.DOTALL).sub(" ", title) - - # Sustituye espacios en blanco por guiones - title = re.compile("\s", re.DOTALL).sub("-", title.strip()) - - # Sustituye espacios en blanco duplicados y saltos de línea - title = re.compile("\-+", re.DOTALL).sub("-", title) - - # Arregla casos especiales - if title.startswith("-"): - title = title[1:] - - if title == "": - title = "-" + str(time.time()) - - return title - - -def remove_htmltags(string): - return re.sub('<[^<]+?>', '', string) - - -def remove_show_from_title(title, show): - # print slugify(title)+" == "+slugify(show) - # Quita el nombre del programa del título - if slugify(title).startswith(slugify(show)): - - # Convierte a unicode primero, o el encoding se pierde - title = unicode(title, "utf-8", "replace") - show = unicode(show, "utf-8", "replace") - title = title[len(show):].strip() - - if title.startswith("-"): - title = title[1:].strip() - - if title == "": - title = str(time.time()) - - # Vuelve a utf-8 - title = title.encode("utf-8", "ignore") - show = show.encode("utf-8", "ignore") - - return title - - -# scrapertools.get_filename_from_url(media_url)[-4:] -def get_filename_from_url(url): - parsed_url = urlparse.urlparse(url) - try: - filename = parsed_url.path - except: - # Si falla es porque la implementación de parsed_url no reconoce los atributos como "path" - if len(parsed_url) >= 4: - filename = parsed_url[2] - else: - filename = "" - - if "/" in filename: - filename = filename.split("/")[-1] - - return filename - - -def get_domain_from_url(url): - parsed_url = urlparse.urlparse(url) - try: - filename = parsed_url.netloc - except: - # Si falla es porque la implementación de parsed_url no reconoce los atributos como "path" - if len(parsed_url) >= 4: - filename = parsed_url[1] - else: - filename = "" - - return filename - - -def get_season_and_episode(title): - """ - Retorna el numero de temporada y de episodio en formato "1x01" obtenido del titulo de un episodio - Ejemplos de diferentes valores para title y su valor devuelto: - "serie 101x1.strm", "s101e1.avi", "t101e1.avi" -> '101x01' - "Name TvShow 1x6.avi" -> '1x06' - "Temp 3 episodio 2.avi" -> '3x02' - "Alcantara season 13 episodie 12.avi" -> '13x12' - "Temp1 capitulo 14" -> '1x14' - "Temporada 1: El origen Episodio 9" -> '' (entre el numero de temporada y los episodios no puede haber otro texto) - "Episodio 25: titulo episodio" -> '' (no existe el numero de temporada) - "Serie X Temporada 1" -> '' (no existe el numero del episodio) - @type title: str - @param title: titulo del episodio de una serie - @rtype: str - @return: Numero de temporada y episodio en formato "1x01" o cadena vacia si no se han encontrado - """ - filename = "" - - patrons = ["(\d+)x(\d+)", "(?:s|t)(\d+)e(\d+)", - "(?:season|temp\w*)\s*(\d+)\s*(?:capitulo|epi\w*)\s*(\d+)"] - - for patron in patrons: - try: - matches = re.compile(patron, re.I).search(title) - if matches: - filename = matches.group(1) + "x" + matches.group(2).zfill(2) - break - except: - pass - - logger.info("'" + title + "' -> '" + filename + "'") - - return filename - - -def get_sha1(cadena): - try: - import hashlib - devuelve = hashlib.sha1(cadena).hexdigest() - except: - import sha - import binascii - devuelve = binascii.hexlify(sha.new(cadena).digest()) - - return devuelve - - -def get_md5(cadena): - try: - import hashlib - devuelve = hashlib.md5(cadena).hexdigest() - except: - import md5 - import binascii - devuelve = binascii.hexlify(md5.new(cadena).digest()) - - return devuelve diff --git a/core/servertools.py b/core/servertools.py index 17cb0dce..9a1ea592 100644 --- a/core/servertools.py +++ b/core/servertools.py @@ -506,8 +506,8 @@ def get_server_json(server_name): def get_server_host(server_name): - from core import scrapertoolsV2 - return [scrapertoolsV2.get_domain_from_url(pattern['url']) for pattern in get_server_json(server_name)['find_videos']['patterns']] + from core import scrapertools + return [scrapertools.get_domain_from_url(pattern['url']) for pattern in get_server_json(server_name)['find_videos']['patterns']] def get_server_controls_settings(server_name): diff --git a/core/support.py b/core/support.py index b03afb5d..b7c5f1bd 100755 --- a/core/support.py +++ b/core/support.py @@ -10,7 +10,7 @@ import urlparse import xbmcaddon from channelselector import thumb -from core import httptools, scrapertoolsV2, servertools, tmdb, channeltools +from core import httptools, scrapertools, servertools, tmdb, channeltools from core.item import Item from lib import unshortenit from platformcode import logger, config @@ -21,7 +21,7 @@ def hdpass_get_servers(item): itemlist = [] data = httptools.downloadpage(item.url).data.replace('\n', '') patron = r']+><\/iframe>' - url = scrapertoolsV2.find_single_match(data, patron).replace("?alta", "") + url = scrapertools.find_single_match(data, patron).replace("?alta", "") url = url.replace("&download=1", "") if 'https' not in url: url = 'https:' + url @@ -37,20 +37,20 @@ def hdpass_get_servers(item): patron_mir = '
(.*?)
' patron_media = r'' - res = scrapertoolsV2.find_single_match(data, patron_res) + res = scrapertools.find_single_match(data, patron_res) itemlist = [] - for res_url, res_video in scrapertoolsV2.find_multiple_matches(res, '([^<]+?)'): + for res_url, res_video in scrapertools.find_multiple_matches(res, '([^<]+?)'): data = httptools.downloadpage(urlparse.urljoin(url, res_url)).data.replace('\n', '') - mir = scrapertoolsV2.find_single_match(data, patron_mir) + mir = scrapertools.find_single_match(data, patron_mir) - for mir_url, srv in scrapertoolsV2.find_multiple_matches(mir, '([^<]+?)'): + for mir_url, srv in scrapertools.find_multiple_matches(mir, '([^<]+?)'): data = httptools.downloadpage(urlparse.urljoin(url, mir_url)).data.replace('\n', '') - for media_label, media_url in scrapertoolsV2.find_multiple_matches(data, patron_media): + for media_label, media_url in scrapertools.find_multiple_matches(data, patron_media): itemlist.append(Item(channel=item.channel, action="play", fulltitle=item.fulltitle, @@ -168,13 +168,13 @@ def scrapeLang(scraped, lang, longtitle): return language, longtitle def cleantitle(title): - cleantitle = scrapertoolsV2.htmlclean(scrapertoolsV2.decodeHtmlentities(title).replace('"', "'").replace('×', 'x').replace('–', '-')).strip() + cleantitle = scrapertools.htmlclean(scrapertools.decodeHtmlentities(title).replace('"', "'").replace('×', 'x').replace('–', '-')).strip() return cleantitle def scrapeBlock(item, args, block, patron, headers, action, pagination, debug, typeContentDict, typeActionDict, blacklist, search, pag, function, lang): itemlist = [] log("scrapeBlock qui", block, patron) - matches = scrapertoolsV2.find_multiple_matches_groups(block, patron) + matches = scrapertools.find_multiple_matches_groups(block, patron) log('MATCHES =', matches) if debug: @@ -214,7 +214,7 @@ def scrapeBlock(item, args, block, patron, headers, action, pagination, debug, t for kk in known_keys: val = match[listGroups.index(kk)] if kk in listGroups else '' if val and (kk == "url" or kk == 'thumb') and 'http' not in val: - val = scrapertoolsV2.find_single_match(item.url, 'https?://[a-z0-9.-]+') + val + val = scrapertools.find_single_match(item.url, 'https?://[a-z0-9.-]+') + val scraped[kk] = val if scraped['season']: @@ -227,7 +227,7 @@ def scrapeBlock(item, args, block, patron, headers, action, pagination, debug, t episode = '' else: episode = re.sub(r'\s-\s|-|x|–|×|×', 'x', scraped['episode']) if scraped['episode'] else '' - second_episode = scrapertoolsV2.find_single_match(episode,'x\d+x(\d+)') + second_episode = scrapertools.find_single_match(episode, 'x\d+x(\d+)') if second_episode: episode = re.sub(r'(\d+x\d+)x\d+',r'\1-', episode) + second_episode.zfill(2) #episode = re.sub(r'\s-\s|-|x|–|×', 'x', scraped['episode']) if scraped['episode'] else '' @@ -257,18 +257,18 @@ def scrapeBlock(item, args, block, patron, headers, action, pagination, debug, t if scraped["plot"]: infolabels['plot'] = plot if scraped['duration']: - matches = scrapertoolsV2.find_multiple_matches(scraped['duration'], + matches = scrapertools.find_multiple_matches(scraped['duration'], r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)') for h, m in matches: scraped['duration'] = int(h) * 60 + int(m) if not matches: - scraped['duration'] = scrapertoolsV2.find_single_match(scraped['duration'], r'(\d+)') + scraped['duration'] = scrapertools.find_single_match(scraped['duration'], r'(\d+)') infolabels['duration'] = int(scraped['duration']) * 60 if scraped['genere']: - genres = scrapertoolsV2.find_multiple_matches(scraped['genere'], '[A-Za-z]+') + genres = scrapertools.find_multiple_matches(scraped['genere'], '[A-Za-z]+') infolabels['genere'] = ", ".join(genres) if scraped["rating"]: - infolabels['rating'] = scrapertoolsV2.decodeHtmlentities(scraped["rating"]) + infolabels['rating'] = scrapertools.decodeHtmlentities(scraped["rating"]) AC = CT = '' if typeContentDict: @@ -379,11 +379,11 @@ def scrape(func): if not data: page = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True, session=item.session) # if url may be changed and channel has findhost to update - if (not page.data or scrapertoolsV2.get_domain_from_url(page.url) != scrapertoolsV2.get_domain_from_url(item.url)) and 'findhost' in func.__globals__: + if (not page.data or scrapertools.get_domain_from_url(page.url) != scrapertools.get_domain_from_url(item.url)) and 'findhost' in func.__globals__: host = func.__globals__['findhost']() from core import jsontools jsontools.update_node(host, func.__module__.split('.')[-1], 'url') - item.url = item.url.replace(scrapertoolsV2.get_domain_from_url(item.url), scrapertoolsV2.get_domain_from_url(host)) + item.url = item.url.replace(scrapertools.get_domain_from_url(item.url), scrapertools.get_domain_from_url(host)) page = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True, session=item.session) data = page.data.replace("'", '"') @@ -394,7 +394,7 @@ def scrape(func): if patronBlock: if debugBlock: regexDbg(item, patronBlock, headers, data) - blocks = scrapertoolsV2.find_multiple_matches_groups(data, patronBlock) + blocks = scrapertools.find_multiple_matches_groups(data, patronBlock) block = "" for bl in blocks: # log(len(blocks),bl) @@ -443,7 +443,7 @@ def scrape(func): if anime: if function == 'episodios' or item.action == 'episodios': autorenumber.renumber(itemlist, item, 'bold') else: autorenumber.renumber(itemlist) - if anime and autorenumber.check(item) == False and not scrapertoolsV2.find_single_match(itemlist[0].title, r'(\d+.\d+)'): + if anime and autorenumber.check(item) == False and not scrapertools.find_single_match(itemlist[0].title, r'(\d+.\d+)'): pass else: if addVideolibrary and (item.infoLabels["title"] or item.fulltitle): @@ -471,7 +471,7 @@ def dooplay_get_links(item, host): data = httptools.downloadpage(item.url).data.replace("'", '"') patron = r'
  • ([^<>]+)(?:([^<>]+))?' - matches = scrapertoolsV2.find_multiple_matches(data, patron) + matches = scrapertools.find_multiple_matches(data, patron) ret = [] @@ -483,7 +483,7 @@ def dooplay_get_links(item, host): "type": type }) dataAdmin = httptools.downloadpage(host + '/wp-admin/admin-ajax.php', post=postData,headers={'Referer': item.url}).data - link = scrapertoolsV2.find_single_match(dataAdmin, "') if not match else match + match = scrapertools.find_single_match(data, r'') if not match else match if not match: from lib import jsunpack try: - data = scrapertoolsV2.find_single_match(data.replace('\n', ''), r"(eval\s?\(function\(p,a,c,k,e,d.*?)") + data = scrapertools.find_single_match(data.replace('\n', ''), r"(eval\s?\(function\(p,a,c,k,e,d.*?)") data = jsunpack.unpack(data) logger.debug("##### play /link/ unpack ##\n%s\n##" % data) except: logger.debug("##### The content is yet unpacked ##\n%s\n##" % data) - data = scrapertoolsV2.find_single_match(data, r'var link(?:\s)?=(?:\s)?"([^"]+)";') + data = scrapertools.find_single_match(data, r'var link(?:\s)?=(?:\s)?"([^"]+)";') data, c = unshortenit.unwrap_30x_only(data) else: data = match @@ -753,7 +753,7 @@ def typo(string, typography=''): if 'submenu' in string: string = u"\u2022\u2022 ".encode('utf-8') + re.sub(r'\ssubmenu','',string) if 'color' in string: - color = scrapertoolsV2.find_single_match(string,'color ([a-z]+)') + color = scrapertools.find_single_match(string, 'color ([a-z]+)') if color == 'kod' or '': color = kod_color string = '[COLOR '+ color +']' + re.sub(r'\scolor\s([a-z]+)','',string) + '[/COLOR]' if 'bold' in string: @@ -785,13 +785,13 @@ def match(item, patron='', patronBlock='', headers='', url='', post=''): log('DATA= ', data) if patronBlock: - block = scrapertoolsV2.find_single_match(data, patronBlock) + block = scrapertools.find_single_match(data, patronBlock) log('BLOCK= ',block) else: block = data if patron: - matches = scrapertoolsV2.find_multiple_matches(block, patron) + matches = scrapertools.find_multiple_matches(block, patron) log('MATCHES= ',matches) return matches, block @@ -899,12 +899,12 @@ def nextPage(itemlist, item, data='', patron='', function_or_level=1, next_page= # If the call is direct, leave it blank action = inspect.stack()[function_or_level][3] if type(function_or_level) == int else function_or_level if next_page == '': - next_page = scrapertoolsV2.find_single_match(data, patron) + next_page = scrapertools.find_single_match(data, patron) if next_page != "": if resub: next_page = re.sub(resub[0], resub[1], next_page) if 'http' not in next_page: - next_page = scrapertoolsV2.find_single_match(item.url, 'https?://[a-z0-9.-]+') + next_page + next_page = scrapertools.find_single_match(item.url, 'https?://[a-z0-9.-]+') + next_page next_page = re.sub('&', '&',next_page) log('NEXT= ', next_page) itemlist.append( diff --git a/platformcode/platformtools.py b/platformcode/platformtools.py index 567c9dbc..88ecd35e 100644 --- a/platformcode/platformtools.py +++ b/platformcode/platformtools.py @@ -20,7 +20,7 @@ import xbmcplugin from channelselector import get_thumb from core import channeltools -from core import trakt_tools, scrapertoolsV2 +from core import trakt_tools, scrapertools from core.item import Item from platformcode import logger, keymaptools from platformcode import unify @@ -799,7 +799,7 @@ def get_seleccion(default_action, opciones, seleccion, video_urls): def calcResolution(option): - match = scrapertoolsV2.find_single_match(option, '([0-9]{2,4})x([0-9]{2,4})') + match = scrapertools.find_single_match(option, '([0-9]{2,4})x([0-9]{2,4})') resolution = False if match: resolution = int(match[0])*int(match[1]) diff --git a/servers/decrypters/zcrypt.py b/servers/decrypters/zcrypt.py index 82be492f..c0e944fc 100644 --- a/servers/decrypters/zcrypt.py +++ b/servers/decrypters/zcrypt.py @@ -2,7 +2,7 @@ # Ringraziamo errmax e dr-z3r0 import re -from core import httptools, scrapertoolsV2 +from core import httptools, scrapertools from platformcode import logger from servers.decrypters import expurl @@ -62,10 +62,10 @@ def get_video_url(page_url, premium=False, user="", password="", video_password= if '/olink/' in url: continue else: idata = httptools.downloadpage(url).data - data = scrapertoolsV2.find_single_match(idata, "]*src=\\'([^'>]*)\\'[^<>]*>") + data = scrapertools.find_single_match(idata, "]*src=\\'([^'>]*)\\'[^<>]*>") #fix by greko inizio if not data: - data = scrapertoolsV2.find_single_match(idata, 'action="(?:[^/]+.*?/[^/]+/([a-zA-Z0-9_]+))">') + data = scrapertools.find_single_match(idata, 'action="(?:[^/]+.*?/[^/]+/([a-zA-Z0-9_]+))">') from lib import unshortenit data, status = unshortenit.unshorten(url) # logger.info("Data - Status zcrypt linkup : [%s] [%s] " %(data, status)) diff --git a/servers/hdload.py b/servers/hdload.py index 46593695..dd09d15c 100644 --- a/servers/hdload.py +++ b/servers/hdload.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from core import httptools, scrapertoolsV2 +from core import httptools, scrapertools from platformcode import config, logger import base64 diff --git a/servers/mixdrop.py b/servers/mixdrop.py index e679dfbc..d94bac1b 100644 --- a/servers/mixdrop.py +++ b/servers/mixdrop.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re from core import httptools -from core import scrapertoolsV2 +from core import scrapertools from platformcode import config, logger from lib import jsunpack @@ -24,9 +24,9 @@ def get_video_url(page_url, premium=False, user="", password="", video_password= data = httptools.downloadpage(page_url).data data = re.sub(r'\n|\t|\r', ' ', data) data = re.sub(r'>\s\s*<', '><', data) - jsCode = scrapertoolsV2.find_single_match(data, r'') + jsCode = scrapertools.find_single_match(data, r'') jsUnpacked = jsunpack.unpack(jsCode) - url = "https://" + scrapertoolsV2.find_single_match(jsUnpacked, r'vsr[^=]*="(?:/)?(/[^"]+)') + url = "https://" + scrapertools.find_single_match(jsUnpacked, r'vsr[^=]*="(?:/)?(/[^"]+)') itemlist.append([".mp4 [MixDrop]", url]) @@ -38,7 +38,7 @@ def get_video_url(page_url, premium=False, user="", password="", video_password= # import simplejson as json # page_url = page_url.replace('/e/', '/f/') + '?download' # data = httptools.downloadpage(page_url).data - # csrf = scrapertoolsV2.find_single_match(data, '') + # csrf = scrapertools.find_single_match(data, '') # postData = {'csrf': csrf, 'a': 'genticket'} # resp = httptools.downloadpage(page_url, post=urllib.urlencode(postData)).data # resp = json.loads(resp) diff --git a/servers/onlystream.py b/servers/onlystream.py index 30de664e..ba1e7b4b 100644 --- a/servers/onlystream.py +++ b/servers/onlystream.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from core import httptools -from core import scrapertoolsV2 +from core import scrapertools from lib import jsunpack from platformcode import config, logger import ast @@ -22,10 +22,10 @@ def get_video_url(page_url, premium=False, user="", password="", video_password= video_urls = [] data = httptools.downloadpage(page_url).data # logger.info(data) - block = scrapertoolsV2.find_single_match(data, r'sources: \[([^\]]+)\]') - sources = scrapertoolsV2.find_multiple_matches(block, r'file:\s*"([^"]+)"(?:,label:\s*"([^"]+)")?') + block = scrapertools.find_single_match(data, r'sources: \[([^\]]+)\]') + sources = scrapertools.find_multiple_matches(block, r'file:\s*"([^"]+)"(?:,label:\s*"([^"]+)")?') if not sources: - sources = scrapertoolsV2.find_multiple_matches(data, r'src:\s*"([^"]+)",\s*type:\s*"[^"]+",[^,]+,\s*label:\s*"([^"]+)"') + sources = scrapertools.find_multiple_matches(data, r'src:\s*"([^"]+)",\s*type:\s*"[^"]+",[^,]+,\s*label:\s*"([^"]+)"') for url, quality in sources: quality = 'auto' if not quality else quality video_urls.append(['.' + url.split('.')[-1] + ' [' + quality + '] [Onlystream]', url]) diff --git a/servers/supervideo.py b/servers/supervideo.py index 355a2762..4ef07daa 100644 --- a/servers/supervideo.py +++ b/servers/supervideo.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from core import httptools -from core import scrapertoolsV2 +from core import scrapertools from lib import jsunpack from platformcode import config, logger import ast @@ -22,10 +22,10 @@ def get_video_url(page_url, premium=False, user="", password="", video_password= video_urls = [] data = httptools.downloadpage(page_url).data logger.info('SUPER DATA= '+data) - code_data = scrapertoolsV2.find_single_match(data, "