Modificaciones gktools y timeout

This commit is contained in:
pipcat
2018-07-02 09:32:17 +02:00
parent 5cf89dd58a
commit ca83da0350
9 changed files with 1068 additions and 209 deletions
+6
View File
@@ -31,6 +31,9 @@ default_headers["Accept-Language"] = "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"
default_headers["Accept-Charset"] = "UTF-8"
default_headers["Accept-Encoding"] = "gzip"
# Tiempo máximo de espera para downloadpage, si no se especifica nada
HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT = None
def get_url_headers(url):
domain_cookies = cj._cookies.get("." + urlparse.urlparse(url)[1], {}).get("/", {})
@@ -122,6 +125,9 @@ def downloadpage(url, post=None, headers=None, timeout=None, follow_redirects=Tr
url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
# Limitar tiempo de descarga si no se ha pasado timeout y hay un valor establecido en la variable global
if timeout is None and HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT is not None: timeout = HTTPTOOLS_DEFAULT_DOWNLOAD_TIMEOUT
logger.info("----------------------------------------------")
logger.info("downloadpage")
logger.info("----------------------------------------------")