stevazi/addon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

KoD 0.8

Browse Source 
- tanti miglioramenti sotto il cofano, supporto iniziale al futuro kodi 19
- Nuova modalità di visualizzazione per episodio successivo
- fixato wstream tramite l'aggiunta della finestra per risolvere il reCaptcha
- aggiunta sezione segnala un problema in Aiuto
- altri fix e migliorie varie a canali e server
This commit is contained in:
marco
2020-02-22 13:36:58 +01:00
parent 82b61df289
commit ca6d5eb56d
121 changed files with 13147 additions and 5448 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/builtins/__init__.py
+12
View File
@@ -0,0 +1,12 @@
from __future__ import absolute_import
import sys
__future_module__ = True
if sys.version_info[0] < 3:
from __builtin__ import *
# Overwrite any old definitions with the equivalent future.builtins ones:
from future.builtins import *
else:
raise ImportError('This package should not be accessible on Python 3. '
'Either you are trying to run from the python-future src folder '
'or your installation of python-future is corrupted.')
lib/cloudscraper/__init__.py
+67 -44
View File
@@ -1,4 +1,6 @@
# https://github.com/VeNoMouS/cloudscraper/tree/master
import logging
import os
import re
import sys
import ssl
@@ -9,6 +11,14 @@ try:
except ImportError:
import copy_reg as copyreg
try:
from HTMLParser import HTMLParser
except ImportError:
if sys.version_info >= (3, 4):
import html
else:
from html.parser import HTMLParser
from copy import deepcopy
from time import sleep
from collections import OrderedDict
@@ -31,13 +41,17 @@ except ImportError:
pass
try:
from urlparse import urlparse
from urlparse import urlparse, urljoin
except ImportError:
from urllib.parse import urlparse
from urllib.parse import urlparse, urljoin
# Add exceptions path
sys.path.append(os.path.join(os.path.dirname(__file__), 'exceptions'))
import cloudflare_exceptions # noqa: E402
# ------------------------------------------------------------------------------- #
__version__ = '1.2.19'
__version__ = '1.2.24'
# ------------------------------------------------------------------------------- #
@@ -91,6 +105,7 @@ class CloudScraper(Session):
'allow_brotli',
True if 'brotli' in sys.modules.keys() else False
)
self.user_agent = User_Agent(
allow_brotli=self.allow_brotli,
browser=kwargs.pop('browser', None)
@@ -107,13 +122,16 @@ class CloudScraper(Session):
# Set a random User-Agent if no custom User-Agent has been set
# ------------------------------------------------------------------------------- #
self.headers = self.user_agent.headers
if not self.cipherSuite:
self.cipherSuite = self.user_agent.cipherSuite
if isinstance(self.cipherSuite, list):
self.cipherSuite = ':'.join(self.cipherSuite)
self.mount(
'https://',
CipherSuiteAdapter(
cipherSuite=':'.join(self.user_agent.cipherSuite)
if not self.cipherSuite else ':'.join(self.cipherSuite)
if isinstance(self.cipherSuite, list) else self.cipherSuite
cipherSuite=self.cipherSuite
)
)
@@ -138,6 +156,20 @@ class CloudScraper(Session):
except ValueError as e:
print("Debug Error: {}".format(getattr(e, 'message', e)))
# ------------------------------------------------------------------------------- #
# Unescape / decode html entities
# ------------------------------------------------------------------------------- #
@staticmethod
def unescape(html_text):
if sys.version_info >= (3, 0):
if sys.version_info >= (3, 4):
return html.unescape(html_text)
return HTMLParser().unescape(html_text)
return HTMLParser().unescape(html_text)
# ------------------------------------------------------------------------------- #
# Decode Brotli on older versions of urllib3 manually
# ------------------------------------------------------------------------------- #
@@ -186,7 +218,7 @@ class CloudScraper(Session):
sys.tracebacklimit = 0
_ = self._solveDepthCnt
self._solveDepthCnt = 0
raise RuntimeError(
raise cloudflare_exceptions.Cloudflare_Loop_Protection(
"!!Loop Protection!! We have tried to solve {} time(s) in a row.".format(_)
)
@@ -269,7 +301,7 @@ class CloudScraper(Session):
def is_Challenge_Request(self, resp):
if self.is_Firewall_Blocked(resp):
sys.tracebacklimit = 0
raise RuntimeError('Cloudflare has blocked this request (Code 1020 Detected).')
raise cloudflare_exceptions.Cloudflare_Block('Cloudflare has blocked this request (Code 1020 Detected).')
if self.is_reCaptcha_Challenge(resp) or self.is_IUAM_Challenge(resp):
return True
@@ -280,17 +312,18 @@ class CloudScraper(Session):
# Try to solve cloudflare javascript challenge.
# ------------------------------------------------------------------------------- #
@staticmethod
def IUAM_Challenge_Response(body, url, interpreter):
def IUAM_Challenge_Response(self, body, url, interpreter):
try:
challengeUUID = re.search(
r'id="challenge-form" action="(?P<challengeUUID>\S+)"',
body, re.M | re.DOTALL
).groupdict().get('challengeUUID', '')
payload = OrderedDict(re.findall(r'name="(r|jschl_vc|pass)"\svalue="(.*?)"', body))
except AttributeError:
sys.tracebacklimit = 0
raise RuntimeError(
raise cloudflare_exceptions.Cloudflare_Error_IUAM(
"Cloudflare IUAM detected, unfortunately we can't extract the parameters correctly."
)
@@ -301,7 +334,7 @@ class CloudScraper(Session):
interpreter
).solveChallenge(body, hostParsed.netloc)
except Exception as e:
raise RuntimeError(
raise cloudflare_exceptions.Cloudflare_Error_IUAM(
'Unable to parse Cloudflare anti-bots page: {}'.format(
getattr(e, 'message', e)
)
@@ -311,7 +344,7 @@ class CloudScraper(Session):
'url': '{}://{}{}'.format(
hostParsed.scheme,
hostParsed.netloc,
challengeUUID
self.unescape(challengeUUID)
),
'data': payload
}
@@ -320,8 +353,7 @@ class CloudScraper(Session):
# Try to solve the reCaptcha challenge via 3rd party.
# ------------------------------------------------------------------------------- #
@staticmethod
def reCaptcha_Challenge_Response(provider, provider_params, body, url):
def reCaptcha_Challenge_Response(self, provider, provider_params, body, url):
try:
payload = re.search(
r'(name="r"\svalue="(?P<r>\S+)"|).*?challenge-form" action="(?P<challengeUUID>\S+)".*?'
@@ -330,7 +362,7 @@ class CloudScraper(Session):
).groupdict()
except (AttributeError):
sys.tracebacklimit = 0
raise RuntimeError(
raise cloudflare_exceptions.Cloudflare_Error_reCaptcha(
"Cloudflare reCaptcha detected, unfortunately we can't extract the parameters correctly."
)
@@ -339,7 +371,7 @@ class CloudScraper(Session):
'url': '{}://{}{}'.format(
hostParsed.scheme,
hostParsed.netloc,
payload.get('challengeUUID', '')
self.unescape(payload.get('challengeUUID', ''))
),
'data': OrderedDict([
('r', payload.get('r', '')),
@@ -377,7 +409,7 @@ class CloudScraper(Session):
if not self.recaptcha or not isinstance(self.recaptcha, dict) or not self.recaptcha.get('provider'):
sys.tracebacklimit = 0
raise RuntimeError(
raise cloudflare_exceptions.Cloudflare_reCaptcha_Provider(
"Cloudflare reCaptcha detected, unfortunately you haven't loaded an anti reCaptcha provider "
"correctly via the 'recaptcha' parameter."
)
@@ -413,7 +445,7 @@ class CloudScraper(Session):
self.delay = delay
except (AttributeError, ValueError):
sys.tracebacklimit = 0
raise RuntimeError("Cloudflare IUAM possibility malformed, issue extracing delay value.")
raise cloudflare_exceptions.Cloudflare_Error_IUAM("Cloudflare IUAM possibility malformed, issue extracing delay value.")
sleep(self.delay)
@@ -473,34 +505,25 @@ class CloudScraper(Session):
return challengeSubmitResponse
else:
cloudflare_kwargs = deepcopy(kwargs)
cloudflare_kwargs['headers'] = updateAttr(
cloudflare_kwargs,
'headers',
{'Referer': challengeSubmitResponse.url}
)
if not urlparse(challengeSubmitResponse.headers['Location']).netloc:
cloudflare_kwargs['headers'] = updateAttr(
cloudflare_kwargs,
'headers',
{'Referer': '{}://{}'.format(urlParsed.scheme, urlParsed.netloc)}
)
return self.request(
resp.request.method,
'{}://{}{}'.format(
urlParsed.scheme,
urlParsed.netloc,
challengeSubmitResponse.headers['Location']
),
**cloudflare_kwargs
redirect_location = urljoin(
challengeSubmitResponse.url,
challengeSubmitResponse.headers['Location']
)
else:
redirectParsed = urlparse(challengeSubmitResponse.headers['Location'])
cloudflare_kwargs['headers'] = updateAttr(
cloudflare_kwargs,
'headers',
{'Referer': '{}://{}'.format(redirectParsed.scheme, redirectParsed.netloc)}
)
return self.request(
resp.request.method,
challengeSubmitResponse.headers['Location'],
**cloudflare_kwargs
)
redirect_location = challengeSubmitResponse.headers['Location']
return self.request(
resp.request.method,
redirect_location,
**cloudflare_kwargs
)
# ------------------------------------------------------------------------------- #
# We shouldn't be here...
@@ -561,7 +584,7 @@ class CloudScraper(Session):
break
else:
sys.tracebacklimit = 0
raise RuntimeError(
raise cloudflare_exceptions.Cloudflare_Error_IUAM(
"Unable to find Cloudflare cookies. Does the site actually "
"have Cloudflare IUAM (I'm Under Attack Mode) enabled?"
)
lib/cloudscraper/exceptions/__init__.py
View File
lib/cloudscraper/exceptions/cloudflare_exceptions.py
+31
View File
@@ -0,0 +1,31 @@
# ------------------------------------------------------------------------------- #
class Cloudflare_Loop_Protection(Exception):
"""
Raise error for recursive depth protection
"""
class Cloudflare_Block(Exception):
"""
Raise error for Cloudflare 1020 block
"""
class Cloudflare_Error_IUAM(Exception):
"""
Raise error for problem extracting IUAM paramters from Cloudflare payload
"""
class Cloudflare_Error_reCaptcha(Exception):
"""
Raise error for problem extracting reCaptcha paramters from Cloudflare payload
"""
class Cloudflare_reCaptcha_Provider(Exception):
"""
Raise error for reCaptcha from Cloudflare, no provider loaded.
"""
lib/cloudscraper/exceptions/reCaptcha_exceptions.py
+49
View File
@@ -0,0 +1,49 @@
# ------------------------------------------------------------------------------- #
class reCaptcha_Service_Unavailable(Exception):
"""
Raise error for external services that cannot be reached
"""
class reCaptcha_Error_From_API(Exception):
"""
Raise error for error from API response.
"""
class reCaptcha_Account_Error(Exception):
"""
Raise error for reCaptcha provider account problem.
"""
class reCaptcha_Timeout(Exception):
"""
Raise error for reCaptcha provider taking too long.
"""
class reCaptcha_Bad_Parameter(NotImplementedError):
"""
Raise error for bad or missing Parameter.
"""
class reCaptcha_Bad_Job_ID(Exception):
"""
Raise error for invalid job id.
"""
class reCaptcha_Report_Error(Exception):
"""
Raise error for reCaptcha provider unable to report bad solve.
"""
class reCaptcha_Import_Error(Exception):
"""
Raise error for reCaptcha, cannot import a module.
"""
lib/cloudscraper/reCaptcha/2captcha.py
+36 -13
View File
@@ -1,13 +1,17 @@
from __future__ import absolute_import
import requests
import reCaptcha_exceptions
try:
import polling
except ImportError:
import sys
sys.tracebacklimit = 0
raise RuntimeError("Please install the python module 'polling' via pip or download it from https://github.com/justiniso/polling/")
raise reCaptcha_exceptions.reCaptcha_Import_Error(
"Please install the python module 'polling' via pip or download it from "
"https://github.com/justiniso/polling/"
)
from . import reCaptcha
@@ -24,7 +28,7 @@ class captchaSolver(reCaptcha):
@staticmethod
def checkErrorStatus(response, request_type):
if response.status_code in [500, 502]:
raise RuntimeError('2Captcha: Server Side Error {}'.format(response.status_code))
raise reCaptcha_exceptions.reCaptcha_Service_Unavailable('2Captcha: Server Side Error {}'.format(response.status_code))
errors = {
'in.php': {
@@ -71,16 +75,23 @@ class captchaSolver(reCaptcha):
}
if response.json().get('status') is False and response.json().get('request') in errors.get(request_type):
raise RuntimeError('{} {}'.format(response.json().get('request'), errors.get(request_type).get(response.json().get('request'))))
raise reCaptcha_exceptions.reCaptcha_Error_From_API(
'{} {}'.format(
response.json().get('request'),
errors.get(request_type).get(response.json().get('request'))
)
)
# ------------------------------------------------------------------------------- #
def reportJob(self, jobID):
if not jobID:
raise RuntimeError("2Captcha: Error bad job id to request reCaptcha.")
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID(
"2Captcha: Error bad job id to request reCaptcha."
)
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get('status') == 1:
if response.ok and response.json().get('status') == 1:
return response
self.checkErrorStatus(response, 'res.php')
@@ -105,7 +116,9 @@ class captchaSolver(reCaptcha):
if response:
return True
else:
raise RuntimeError("2Captcha: Error - Failed to report bad reCaptcha solve.")
raise reCaptcha_exceptions.reCaptcha_Report_Error(
"2Captcha: Error - Failed to report bad reCaptcha solve."
)
# ------------------------------------------------------------------------------- #
@@ -114,7 +127,7 @@ class captchaSolver(reCaptcha):
raise RuntimeError("2Captcha: Error bad job id to request reCaptcha.")
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get('status') == 1:
if response.ok and response.json().get('status') == 1:
return response
self.checkErrorStatus(response, 'res.php')
@@ -139,13 +152,15 @@ class captchaSolver(reCaptcha):
if response:
return response.json().get('request')
else:
raise RuntimeError("2Captcha: Error failed to solve reCaptcha.")
raise reCaptcha_exceptions.reCaptcha_Timeout(
"2Captcha: Error failed to solve reCaptcha."
)
# ------------------------------------------------------------------------------- #
def requestSolve(self, site_url, site_key):
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get("status") == 1 and response.json().get('request'):
if response.ok and response.json().get("status") == 1 and response.json().get('request'):
return response
self.checkErrorStatus(response, 'in.php')
@@ -173,7 +188,9 @@ class captchaSolver(reCaptcha):
if response:
return response.json().get('request')
else:
raise RuntimeError('2Captcha: Error no job id was returned.')
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID(
'2Captcha: Error no job id was returned.'
)
# ------------------------------------------------------------------------------- #
@@ -181,7 +198,9 @@ class captchaSolver(reCaptcha):
jobID = None
if not reCaptchaParams.get('api_key'):
raise ValueError("2Captcha: Missing api_key parameter.")
raise reCaptcha_exceptions.reCaptcha_Bad_Parameter(
"2Captcha: Missing api_key parameter."
)
self.api_key = reCaptchaParams.get('api_key')
@@ -196,9 +215,13 @@ class captchaSolver(reCaptcha):
if jobID:
self.reportJob(jobID)
except polling.TimeoutException:
raise RuntimeError("2Captcha: reCaptcha solve took to long and also failed reporting the job.")
raise reCaptcha_exceptions.reCaptcha_Timeout(
"2Captcha: reCaptcha solve took to long and also failed reporting the job the job id {}.".format(jobID)
)
raise RuntimeError("2Captcha: reCaptcha solve took to long to execute, aborting.")
raise reCaptcha_exceptions.reCaptcha_Timeout(
"2Captcha: reCaptcha solve took to long to execute job id {}, aborting.".format(jobID)
)
# ------------------------------------------------------------------------------- #
lib/cloudscraper/reCaptcha/9kw.py
+202
View File
@@ -0,0 +1,202 @@
from __future__ import absolute_import
import re
import requests
import reCaptcha_exceptions
try:
import polling
except ImportError:
import sys
sys.tracebacklimit = 0
raise reCaptcha_exceptions.reCaptcha_Import_Error(
"Please install the python module 'polling' via pip or download it from "
"https://github.com/justiniso/polling/"
)
from . import reCaptcha
class captchaSolver(reCaptcha):
def __init__(self):
super(captchaSolver, self).__init__('9kw')
self.host = 'https://www.9kw.eu/index.cgi'
self.maxtimeout = 180
self.session = requests.Session()
# ------------------------------------------------------------------------------- #
@staticmethod
def checkErrorStatus(response):
if response.status_code in [500, 502]:
raise reCaptcha_exceptions.reCaptcha_Service_Unavailable(
'9kw: Server Side Error {}'.format(response.status_code)
)
error_codes = {
1: 'No API Key available.',
2: 'No API key found.',
3: 'No active API key found.',
4: 'API Key has been disabled by the operator. ',
5: 'No user found.',
6: 'No data found.',
7: 'Found No ID.',
8: 'found No captcha.',
9: 'No image found.',
10: 'Image size not allowed.',
11: 'credit is not sufficient.',
12: 'what was done.',
13: 'No answer contain.',
14: 'Captcha already been answered.',
15: 'Captcha to quickly filed.',
16: 'JD check active.',
17: 'Unknown problem.',
18: 'Found No ID.',
19: 'Incorrect answer.',
20: 'Do not timely filed (Incorrect UserID).',
21: 'Link not allowed.',
22: 'Prohibited submit.',
23: 'Entering prohibited.',
24: 'Too little credit.',
25: 'No entry found.',
26: 'No Conditions accepted.',
27: 'No coupon code found in the database.',
28: 'Already unused voucher code.',
29: 'maxTimeout under 60 seconds.',
30: 'User not found.',
31: 'An account is not yet 24 hours in system.',
32: 'An account does not have the full rights.',
33: 'Plugin needed a update.',
34: 'No HTTPS allowed.',
35: 'No HTTP allowed.',
36: 'Source not allowed.',
37: 'Transfer denied.',
38: 'Incorrect answer without space',
39: 'Incorrect answer with space',
40: 'Incorrect answer with not only numbers',
41: 'Incorrect answer with not only A-Z, a-z',
42: 'Incorrect answer with not only 0-9, A-Z, a-z',
43: 'Incorrect answer with not only [0-9,- ]',
44: 'Incorrect answer with not only [0-9A-Za-z,- ]',
45: 'Incorrect answer with not only coordinates',
46: 'Incorrect answer with not only multiple coordinates',
47: 'Incorrect answer with not only data',
48: 'Incorrect answer with not only rotate number',
49: 'Incorrect answer with not only text',
50: 'Incorrect answer with not only text and too short',
51: 'Incorrect answer with not enough chars',
52: 'Incorrect answer with too many chars',
53: 'Incorrect answer without no or yes',
54: 'Assignment was not found.'
}
if response.text.startswith('{'):
if response.json().get('error'):
raise reCaptcha_exceptions.reCaptcha_Error_From_API(error_codes.get(int(response.json().get('error'))))
else:
error_code = int(re.search(r'^00(?P<error_code>\d+)', response.text).groupdict().get('error_code', 0))
if error_code:
raise reCaptcha_exceptions.reCaptcha_Error_From_API(error_codes.get(error_code))
# ------------------------------------------------------------------------------- #
def requestJob(self, jobID):
if not jobID:
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID(
"9kw: Error bad job id to request reCaptcha against."
)
def _checkRequest(response):
if response.ok and response.json().get('answer') != 'NO DATA':
return response
self.checkErrorStatus(response)
return None
response = polling.poll(
lambda: self.session.get(
self.host,
params={
'apikey': self.api_key,
'action': 'usercaptchacorrectdata',
'id': jobID,
'info': 1,
'json': 1
}
),
check_success=_checkRequest,
step=10,
timeout=(self.maxtimeout + 10)
)
if response:
return response.json().get('answer')
else:
raise reCaptcha_exceptions.reCaptcha_Timeout("9kw: Error failed to solve reCaptcha.")
# ------------------------------------------------------------------------------- #
def requestSolve(self, site_url, site_key):
def _checkRequest(response):
if response.ok and response.text.startswith('{') and response.json().get('captchaid'):
return response
self.checkErrorStatus(response)
return None
response = polling.poll(
lambda: self.session.post(
self.host,
data={
'apikey': self.api_key,
'action': 'usercaptchaupload',
'interactive': 1,
'file-upload-01': site_key,
'oldsource': 'recaptchav2',
'pageurl': site_url,
'maxtimeout': self.maxtimeout,
'json': 1
},
allow_redirects=False
),
check_success=_checkRequest,
step=5,
timeout=(self.maxtimeout + 10)
)
if response:
return response.json().get('captchaid')
else:
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID('9kw: Error no valid job id was returned.')
# ------------------------------------------------------------------------------- #
def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
jobID = None
if not reCaptchaParams.get('api_key'):
raise reCaptcha_exceptions.reCaptcha_Bad_Parameter("9kw: Missing api_key parameter.")
self.api_key = reCaptchaParams.get('api_key')
if reCaptchaParams.get('maxtimeout'):
self.maxtimeout = reCaptchaParams.get('maxtimeout')
if reCaptchaParams.get('proxy'):
self.session.proxies = reCaptchaParams.get('proxies')
try:
jobID = self.requestSolve(site_url, site_key)
return self.requestJob(jobID)
except polling.TimeoutException:
raise reCaptcha_exceptions.reCaptcha_Timeout(
"9kw: reCaptcha solve took to long to execute 'captchaid' {}, aborting.".format(jobID)
)
# ------------------------------------------------------------------------------- #
captchaSolver()
lib/cloudscraper/reCaptcha/anticaptcha.py
+13 -3
View File
@@ -1,12 +1,16 @@
from __future__ import absolute_import
import sys
import reCaptcha_exceptions
try:
from python_anticaptcha import AnticaptchaClient, NoCaptchaTaskProxylessTask
except ImportError:
sys.tracebacklimit = 0
raise RuntimeError("Please install the python module 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha")
raise reCaptcha_exceptions.reCaptcha_Import_Error(
"Please install the python module 'python_anticaptcha' via pip or download it from "
"https://github.com/ad-m/python-anticaptcha"
)
from . import reCaptcha
@@ -16,9 +20,11 @@ class captchaSolver(reCaptcha):
def __init__(self):
super(captchaSolver, self).__init__('anticaptcha')
# ------------------------------------------------------------------------------- #
def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
if not reCaptchaParams.get('api_key'):
raise ValueError("reCaptcha provider 'anticaptcha' was not provided an 'api_key' parameter.")
raise reCaptcha_exceptions.reCaptcha_Bad_Parameter("anticaptcha: Missing api_key parameter.")
client = AnticaptchaClient(reCaptchaParams.get('api_key'))
@@ -29,10 +35,14 @@ class captchaSolver(reCaptcha):
if not hasattr(client, 'createTaskSmee'):
sys.tracebacklimit = 0
raise RuntimeError("Please upgrade 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha")
raise reCaptcha_exceptions.reCaptcha_Import_Error(
"Please upgrade 'python_anticaptcha' via pip or download it from https://github.com/ad-m/python-anticaptcha"
)
job = client.createTaskSmee(task)
return job.get_solution_response()
# ------------------------------------------------------------------------------- #
captchaSolver()
lib/cloudscraper/reCaptcha/deathbycaptcha.py
+41 -21
View File
@@ -2,13 +2,17 @@ from __future__ import absolute_import
import json
import requests
import reCaptcha_exceptions
try:
import polling
except ImportError:
import sys
sys.tracebacklimit = 0
raise RuntimeError("Please install the python module 'polling' via pip or download it from https://github.com/justiniso/polling/")
raise reCaptcha_exceptions.reCaptcha_Import_Error(
"Please install the python module 'polling' via pip or download it from "
"https://github.com/justiniso/polling/"
)
from . import reCaptcha
@@ -20,7 +24,7 @@ class captchaSolver(reCaptcha):
self.host = 'http://api.dbcapi.me/api'
self.session = requests.Session()
# ------------------------------------------------------------------------------- #
# ------------------------------------------------------------------------------- #
@staticmethod
def checkErrorStatus(response):
@@ -34,21 +38,21 @@ class captchaSolver(reCaptcha):
)
if response.status_code in errors:
raise RuntimeError(errors.get(response.status_code))
raise reCaptcha_exceptions.reCaptcha_Service_Unavailable(errors.get(response.status_code))
# ------------------------------------------------------------------------------- #
# ------------------------------------------------------------------------------- #
def login(self, username, password):
self.username = username
self.password = password
def _checkRequest(response):
if response.status_code == 200:
if response.ok:
if response.json().get('is_banned'):
raise RuntimeError('DeathByCaptcha: Your account is banned.')
raise reCaptcha_exceptions.reCaptcha_Account_Error('DeathByCaptcha: Your account is banned.')
if response.json().get('balanace') == 0:
raise RuntimeError('DeathByCaptcha: insufficient credits.')
raise reCaptcha_exceptions.reCaptcha_Account_Error('DeathByCaptcha: insufficient credits.')
return response
@@ -72,11 +76,13 @@ class captchaSolver(reCaptcha):
self.debugRequest(response)
# ------------------------------------------------------------------------------- #
# ------------------------------------------------------------------------------- #
def reportJob(self, jobID):
if not jobID:
raise RuntimeError("DeathByCaptcha: Error bad job id to report failed reCaptcha.")
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID(
"DeathByCaptcha: Error bad job id to report failed reCaptcha."
)
def _checkRequest(response):
if response.status_code == 200:
@@ -103,16 +109,20 @@ class captchaSolver(reCaptcha):
if response:
return True
else:
raise RuntimeError("DeathByCaptcha: Error report failed reCaptcha.")
raise reCaptcha_exceptions.reCaptcha_Report_Error(
"DeathByCaptcha: Error report failed reCaptcha."
)
# ------------------------------------------------------------------------------- #
# ------------------------------------------------------------------------------- #
def requestJob(self, jobID):
if not jobID:
raise RuntimeError("DeathByCaptcha: Error bad job id to request reCaptcha.")
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID(
"DeathByCaptcha: Error bad job id to request reCaptcha."
)
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get('text'):
if response.ok and response.json().get('text'):
return response
self.checkErrorStatus(response)
@@ -132,13 +142,15 @@ class captchaSolver(reCaptcha):
if response:
return response.json().get('text')
else:
raise RuntimeError("DeathByCaptcha: Error failed to solve reCaptcha.")
raise reCaptcha_exceptions.reCaptcha_Timeout(
"DeathByCaptcha: Error failed to solve reCaptcha."
)
# ------------------------------------------------------------------------------- #
# ------------------------------------------------------------------------------- #
def requestSolve(self, site_url, site_key):
def _checkRequest(response):
if response.status_code in [200, 303] and response.json().get("is_correct") and response.json().get('captcha'):
if response.ok and response.json().get("is_correct") and response.json().get('captcha'):
return response
self.checkErrorStatus(response)
@@ -168,16 +180,20 @@ class captchaSolver(reCaptcha):
if response:
return response.json().get('captcha')
else:
raise RuntimeError('DeathByCaptcha: Error no job id was returned.')
raise reCaptcha_exceptions.reCaptcha_Bad_Job_ID(
'DeathByCaptcha: Error no job id was returned.'
)
# ------------------------------------------------------------------------------- #
# ------------------------------------------------------------------------------- #
def getCaptchaAnswer(self, site_url, site_key, reCaptchaParams):
jobID = None
for param in ['username', 'password']:
if not reCaptchaParams.get(param):
raise ValueError("DeathByCaptcha: Missing '{}' parameter.".format(param))
raise reCaptcha_exceptions.reCaptcha_Bad_Parameter(
"DeathByCaptcha: Missing '{}' parameter.".format(param)
)
setattr(self, param, reCaptchaParams.get(param))
if reCaptchaParams.get('proxy'):
@@ -191,9 +207,13 @@ class captchaSolver(reCaptcha):
if jobID:
self.reportJob(jobID)
except polling.TimeoutException:
raise RuntimeError("DeathByCaptcha: reCaptcha solve took to long and also failed reporting the job.")
raise reCaptcha_exceptions.reCaptcha_Timeout(
"DeathByCaptcha: reCaptcha solve took to long and also failed reporting the job id {}.".format(jobID)
)
raise RuntimeError("DeathByCaptcha: reCaptcha solve took to long to execute, aborting.")
raise reCaptcha_exceptions.reCaptcha_Timeout(
"DeathByCaptcha: reCaptcha solve took to long to execute job id {}, aborting.".format(jobID)
)
# ------------------------------------------------------------------------------- #
lib/cloudscraper/user_agent/__init__.py
+6 -5
View File
@@ -47,7 +47,7 @@ class User_Agent():
for browser in user_agents:
for release in user_agents[browser]['releases']:
for platform in ['mobile', 'desktop']:
if re.search(self.custom, ' '.join(user_agents[browser]['releases'][release]['User-Agent'][platform])):
if re.search(re.escape(self.custom), ' '.join(user_agents[browser]['releases'][release]['User-Agent'][platform])):
self.browser = browser
self.loadHeaders(user_agents, release)
self.headers['User-Agent'] = self.custom
@@ -74,10 +74,11 @@ class User_Agent():
sys.tracebacklimit = 0
raise RuntimeError("Sorry you can't have mobile and desktop disabled at the same time.")
user_agents = json.load(
open(os.path.join(os.path.dirname(__file__), 'browsers.json'), 'r'),
object_pairs_hook=OrderedDict
)
with open(os.path.join(os.path.dirname(__file__), 'browsers.json'), 'r') as fp:
user_agents = json.load(
fp,
object_pairs_hook=OrderedDict
)
if self.custom:
if not self.tryMatchCustom(user_agents):
lib/cloudscraper/user_agent/browsers.json
+10
View File
@@ -13,9 +13,14 @@
"TLS_CHACHA20_POLY1305_SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305",
"ECDHE-RSA-AES256-SHA",
"AES128-GCM-SHA256",
"AES256-GCM-SHA384",
"AES128-SHA",
"AES256-SHA"
],
"releases": {
@@ -12814,10 +12819,15 @@
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-CHACHA20-POLY1305",
"ECDHE-RSA-CHACHA20-POLY1305",
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-RSA-AES128-SHA",
"DHE-RSA-AES256-SHA",
"AES128-SHA",
"AES256-SHA"
],
"releases": {
lib/future/backports/email/base64mime.py
+1
View File
@@ -28,6 +28,7 @@ from __future__ import division
from __future__ import absolute_import
from future.builtins import range
from future.builtins import bytes
from future.builtins import str
__all__ = [
'body_decode',
lib/future/backports/test/pystone.py
Regular → Executable
View File
lib/future/backports/urllib/error.py
+2 -2
View File
@@ -11,9 +11,9 @@ an application may want to handle an exception like a regular
response.
"""
from __future__ import absolute_import, division, unicode_literals
from ... import standard_library
from future import standard_library
from . import response as urllib_response
from future.backports.urllib import response as urllib_response
__all__ = ['URLError', 'HTTPError', 'ContentTooShortError']
lib/future/backports/urllib/parse.py
+4 -4
View File
@@ -87,7 +87,7 @@ def clear_cache():
# decoding and encoding. If valid use cases are
# presented, we may relax this by using latin-1
# decoding internally for 3.3
_implicit_encoding = 'utf8'
_implicit_encoding = 'ascii'
_implicit_errors = 'strict'
def _noop(obj):
@@ -122,7 +122,7 @@ class _ResultMixinStr(object):
"""Standard approach to encoding parsed results from str to bytes"""
__slots__ = ()
def encode(self, encoding='utf8', errors='strict'):
def encode(self, encoding='ascii', errors='strict'):
return self._encoded_counterpart(*(x.encode(encoding, errors) for x in self))
@@ -130,7 +130,7 @@ class _ResultMixinBytes(object):
"""Standard approach to decoding parsed results from bytes to str"""
__slots__ = ()
def decode(self, encoding='utf8', errors='strict'):
def decode(self, encoding='ascii', errors='strict'):
return self._decoded_counterpart(*(x.decode(encoding, errors) for x in self))
@@ -730,7 +730,7 @@ def quote_from_bytes(bs, safe='/'):
###
if isinstance(safe, str):
# Normalize 'safe' by converting to bytes and removing non-ASCII chars
safe = str(safe).encode('utf8', 'ignore')
safe = str(safe).encode('ascii', 'ignore')
else:
### For Python-Future:
safe = bytes(safe)
lib/future/backports/urllib/request.py
+9 -19
View File
@@ -827,7 +827,7 @@ class ProxyHandler(BaseHandler):
if user and password:
user_pass = '%s:%s' % (unquote(user),
unquote(password))
creds = base64.b64encode(user_pass.encode()).decode("utf8")
creds = base64.b64encode(user_pass.encode()).decode("ascii")
req.add_header('Proxy-authorization', 'Basic ' + creds)
hostport = unquote(hostport)
req.set_proxy(hostport, proxy_type)
@@ -977,7 +977,7 @@ class AbstractBasicAuthHandler(object):
user, pw = self.passwd.find_user_password(realm, host)
if pw is not None:
raw = "%s:%s" % (user, pw)
auth = "Basic " + base64.b64encode(raw.encode()).decode("utf8")
auth = "Basic " + base64.b64encode(raw.encode()).decode("ascii")
if req.headers.get(self.auth_header, None) == auth:
return None
req.add_unredirected_header(self.auth_header, auth)
@@ -1080,7 +1080,7 @@ class AbstractDigestAuthHandler(object):
# authentication, and to provide some message integrity protection.
# This isn't a fabulous effort, but it's probably Good Enough.
s = "%s:%s:%s:" % (self.nonce_count, nonce, time.ctime())
b = s.encode("utf8") + _randombytes(8)
b = s.encode("ascii") + _randombytes(8)
dig = hashlib.sha1(b).hexdigest()
return dig[:16]
@@ -1147,9 +1147,9 @@ class AbstractDigestAuthHandler(object):
def get_algorithm_impls(self, algorithm):
# lambdas assume digest modules are imported at the top level
if algorithm == 'MD5':
H = lambda x: hashlib.md5(x.encode("utf8")).hexdigest()
H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest()
elif algorithm == 'SHA':
H = lambda x: hashlib.sha1(x.encode("utf8")).hexdigest()
H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest()
# XXX MD5-sess
KD = lambda s, d: H("%s:%s" % (s, d))
return H, KD
@@ -1829,13 +1829,13 @@ class URLopener(object):
if proxy_passwd:
proxy_passwd = unquote(proxy_passwd)
proxy_auth = base64.b64encode(proxy_passwd.encode()).decode('utf8')
proxy_auth = base64.b64encode(proxy_passwd.encode()).decode('ascii')
else:
proxy_auth = None
if user_passwd:
user_passwd = unquote(user_passwd)
auth = base64.b64encode(user_passwd.encode()).decode('utf8')
auth = base64.b64encode(user_passwd.encode()).decode('ascii')
else:
auth = None
http_conn = connection_factory(host)
@@ -2040,7 +2040,7 @@ class URLopener(object):
msg.append('Content-type: %s' % type)
if encoding == 'base64':
# XXX is this encoding/decoding ok?
data = base64.decodebytes(data.encode('utf8')).decode('latin-1')
data = base64.decodebytes(data.encode('ascii')).decode('latin-1')
else:
data = unquote(data)
msg.append('Content-Length: %d' % len(data))
@@ -2498,17 +2498,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings):
if sys.platform == 'darwin':
try:
from _scproxy import _get_proxy_settings, _get_proxies
except:
try:
# By default use environment variables
_get_proxy_settings = getproxies_environment
_get_proxies = proxy_bypass_environment
getproxies = getproxies_environment
proxy_bypass = proxy_bypass_environment
except:
pass
from _scproxy import _get_proxy_settings, _get_proxies
def proxy_bypass_macosx_sysconf(host):
proxy_settings = _get_proxy_settings()
lib/future/backports/urllib/robotparser.py
+3 -3
View File
@@ -1,5 +1,5 @@
from __future__ import absolute_import, division, unicode_literals
from ...builtins import str
from future.builtins import str
""" robotparser.py
Copyright (C) 2000 Bastian Kleineidam
@@ -13,8 +13,8 @@ from ...builtins import str
"""
# Was: import urllib.parse, urllib.request
from .. import urllib
from . import parse as _parse, request as _request
from future.backports import urllib
from future.backports.urllib import parse as _parse, request as _request
urllib.parse = _parse
urllib.request = _request
lib/future/builtins/newround.py
+10 -7
View File
@@ -2,6 +2,7 @@
``python-future``: pure Python implementation of Python 3 round().
"""
from __future__ import division
from future.utils import PYPY, PY26, bind_method
# Use the decimal module for simplicity of implementation (and
@@ -29,8 +30,6 @@ def newround(number, ndigits=None):
if hasattr(number, '__round__'):
return number.__round__(ndigits)
if ndigits < 0:
raise NotImplementedError('negative ndigits not supported yet')
exponent = Decimal('10') ** (-ndigits)
if PYPY:
@@ -42,15 +41,19 @@ def newround(number, ndigits=None):
d = number
else:
if not PY26:
d = Decimal.from_float(number).quantize(exponent,
rounding=ROUND_HALF_EVEN)
d = Decimal.from_float(number)
else:
d = from_float_26(number).quantize(exponent, rounding=ROUND_HALF_EVEN)
d = from_float_26(number)
if ndigits < 0:
result = newround(d / exponent) * exponent
else:
result = d.quantize(exponent, rounding=ROUND_HALF_EVEN)
if return_int:
return int(d)
return int(result)
else:
return float(d)
return float(result)
### From Python 2.7's decimal.py. Only needed to support Py2.6:
lib/future/moves/tkinter/filedialog.py
+6
View File
@@ -10,3 +10,9 @@ else:
except ImportError:
raise ImportError('The FileDialog module is missing. Does your Py2 '
'installation include tkinter?')
try:
from tkFileDialog import *
except ImportError:
raise ImportError('The tkFileDialog module is missing. Does your Py2 '
'installation include tkinter?')
lib/future/standard_library/__init__.py
+43 -77
View File
@@ -450,63 +450,35 @@ def install_aliases():
# if hasattr(install_aliases, 'run_already'):
# return
for (newmodname, newobjname, oldmodname, oldobjname) in MOVES:
try:
__import__(newmodname)
# We look up the module in sys.modules because __import__ just returns the
# top-level package:
newmod = sys.modules[newmodname]
# newmod.__future_module__ = True
__import__(newmodname)
# We look up the module in sys.modules because __import__ just returns the
# top-level package:
newmod = sys.modules[newmodname]
# newmod.__future_module__ = True
__import__(oldmodname)
oldmod = sys.modules[oldmodname]
__import__(oldmodname)
oldmod = sys.modules[oldmodname]
obj = getattr(oldmod, oldobjname)
setattr(newmod, newobjname, obj)
except:
try:
flog.warning('*** FUTURE ERROR in module %s %s ' % (str(oldmod), str(oldobjname)))
except:
pass
obj = getattr(oldmod, oldobjname)
setattr(newmod, newobjname, obj)
# Hack for urllib so it appears to have the same structure on Py2 as on Py3
try:
import urllib
from future.backports.urllib import response
urllib.response = response
sys.modules['urllib.response'] = response
from future.backports.urllib import parse
urllib.parse = parse
sys.modules['urllib.parse'] = parse
from future.backports.urllib import error
urllib.error = error
sys.modules['urllib.error'] = error
except ImportError:
try:
flog.warning('*** FUTURE ERROR importing URLLIB.response, parse, error')
urllib.response = urllib
sys.modules['urllib.response'] = urllib
urllib.parse = urllib
sys.modules['urllib.parse'] = urllib
urllib.error = urllib
sys.modules['urllib.error'] = urllib
except:
pass
try:
from future.backports.urllib import request
urllib.request = request
sys.modules['urllib.request'] = request
from future.backports.urllib import robotparser
urllib.robotparser = robotparser
sys.modules['urllib.robotparser'] = robotparser
except ImportError:
try:
flog.warning('*** FUTURE ERROR importing URLLIB.Request')
urllib.request = urllib
sys.modules['urllib.request'] = urllib
urllib.robotparser = urllib
sys.modules['urllib.robotparser'] = urllib
except:
pass
import urllib
from future.backports.urllib import request
from future.backports.urllib import response
from future.backports.urllib import parse
from future.backports.urllib import error
from future.backports.urllib import robotparser
urllib.request = request
urllib.response = response
urllib.parse = parse
urllib.error = error
urllib.robotparser = robotparser
sys.modules['urllib.request'] = request
sys.modules['urllib.response'] = response
sys.modules['urllib.parse'] = parse
sys.modules['urllib.error'] = error
sys.modules['urllib.robotparser'] = robotparser
# Patch the test module so it appears to have the same structure on Py2 as on Py3
try:
@@ -518,11 +490,8 @@ def install_aliases():
except ImportError:
pass
else:
try:
test.support = support
sys.modules['test.support'] = support
except:
pass
test.support = support
sys.modules['test.support'] = support
# Patch the dbm module so it appears to have the same structure on Py2 as on Py3
try:
@@ -530,26 +499,23 @@ def install_aliases():
except ImportError:
pass
else:
from future.moves.dbm import dumb
dbm.dumb = dumb
sys.modules['dbm.dumb'] = dumb
try:
from future.moves.dbm import dumb
dbm.dumb = dumb
sys.modules['dbm.dumb'] = dumb
try:
from future.moves.dbm import gnu
except ImportError:
pass
else:
dbm.gnu = gnu
sys.modules['dbm.gnu'] = gnu
try:
from future.moves.dbm import ndbm
except ImportError:
pass
else:
dbm.ndbm = ndbm
sys.modules['dbm.ndbm'] = ndbm
except:
flog.warning('*** FUTURE ERROR importing MOVES.dbm')
from future.moves.dbm import gnu
except ImportError:
pass
else:
dbm.gnu = gnu
sys.modules['dbm.gnu'] = gnu
try:
from future.moves.dbm import ndbm
except ImportError:
pass
else:
dbm.ndbm = ndbm
sys.modules['dbm.ndbm'] = ndbm
# install_aliases.run_already = True
lib/future/utils/__init__.py
+2 -2
View File
@@ -527,9 +527,9 @@ def implements_iterator(cls):
return cls
if PY3:
get_next = lambda x: x.next
else:
get_next = lambda x: x.__next__
else:
get_next = lambda x: x.next
def encode_filename(filename):
lib/generictools.py
+6 -2
View File
@@ -11,7 +11,10 @@ import re
import os
import sys
import urllib
import urlparse
try:
import urlparse
except:
import urllib.parse as urlparse
import datetime
import time
import traceback
@@ -161,7 +164,8 @@ def update_title(item):
item.channel = new_item.channel #Restuaramos el nombre del canal, por si lo habíamos cambiado
if item.tmdb_stat == True:
if new_item.contentSerieName: #Si es serie...
if config.get_setting("filter_languages", item.channel) >= 0:
filter_languages = config.get_setting("filter_languages", item.channel)
if filter_languages and filter_languages >= 0:
item.title_from_channel = new_item.contentSerieName #Guardo el título incial para Filtertools
item.contentSerieName = new_item.contentSerieName #Guardo el título incial para Filtertools
else:
lib/githash.py
-1
View File
@@ -9,7 +9,6 @@ has been tested with Python2.7 and Python 3.4.
from __future__ import print_function
import argparse
import os
import stat
import sys
lib/httplib2/__init__.py
-2231
View File
File diff suppressed because it is too large Load Diff
lib/httplib2/py2/__init__.py
+2285
View File
File diff suppressed because it is too large Load Diff
lib/httplib2/cacerts.txt → lib/httplib2/py2/cacerts.txt
View File
lib/httplib2/certs.py → lib/httplib2/py2/certs.py
View File
lib/httplib2/iri2uri.py → lib/httplib2/py2/iri2uri.py
View File
lib/httplib2/socks.py → lib/httplib2/py2/socks.py
+9 -1
View File
@@ -238,7 +238,15 @@ class socksocket(socket.socket):
headers - Additional or modified headers for the proxy connect
request.
"""
self.__proxy = (proxytype, addr, port, rdns, username, password, headers)
self.__proxy = (
proxytype,
addr,
port,
rdns,
username.encode() if username else None,
password.encode() if password else None,
headers,
)
def __negotiatesocks5(self, destaddr, destport):
"""__negotiatesocks5(self,destaddr,destport)
lib/httplib2/py3/__init__.py
+2077
View File
File diff suppressed because it is too large Load Diff
lib/httplib2/py3/cacerts.txt
+2197
View File
File diff suppressed because it is too large Load Diff
lib/httplib2/py3/certs.py
+42
View File
@@ -0,0 +1,42 @@
"""Utilities for certificate management."""
import os
certifi_available = False
certifi_where = None
try:
from certifi import where as certifi_where
certifi_available = True
except ImportError:
pass
custom_ca_locater_available = False
custom_ca_locater_where = None
try:
from ca_certs_locater import get as custom_ca_locater_where
custom_ca_locater_available = True
except ImportError:
pass
BUILTIN_CA_CERTS = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "cacerts.txt"
)
def where():
env = os.environ.get("HTTPLIB2_CA_CERTS")
if env is not None:
if os.path.isfile(env):
return env
else:
raise RuntimeError("Environment variable HTTPLIB2_CA_CERTS not a valid file")
if custom_ca_locater_available:
return custom_ca_locater_where()
if certifi_available:
return certifi_where()
return BUILTIN_CA_CERTS
if __name__ == "__main__":
print(where())
lib/httplib2/py3/iri2uri.py
+124
View File
@@ -0,0 +1,124 @@
# -*- coding: utf-8 -*-
"""Converts an IRI to a URI."""
__author__ = "Joe Gregorio (joe@bitworking.org)"
__copyright__ = "Copyright 2006, Joe Gregorio"
__contributors__ = []
__version__ = "1.0.0"
__license__ = "MIT"
import urllib.parse
# Convert an IRI to a URI following the rules in RFC 3987
#
# The characters we need to enocde and escape are defined in the spec:
#
# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD
# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF
# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD
# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD
# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD
# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD
# / %xD0000-DFFFD / %xE1000-EFFFD
escape_range = [
(0xA0, 0xD7FF),
(0xE000, 0xF8FF),
(0xF900, 0xFDCF),
(0xFDF0, 0xFFEF),
(0x10000, 0x1FFFD),
(0x20000, 0x2FFFD),
(0x30000, 0x3FFFD),
(0x40000, 0x4FFFD),
(0x50000, 0x5FFFD),
(0x60000, 0x6FFFD),
(0x70000, 0x7FFFD),
(0x80000, 0x8FFFD),
(0x90000, 0x9FFFD),
(0xA0000, 0xAFFFD),
(0xB0000, 0xBFFFD),
(0xC0000, 0xCFFFD),
(0xD0000, 0xDFFFD),
(0xE1000, 0xEFFFD),
(0xF0000, 0xFFFFD),
(0x100000, 0x10FFFD),
]
def encode(c):
retval = c
i = ord(c)
for low, high in escape_range:
if i < low:
break
if i >= low and i <= high:
retval = "".join(["%%%2X" % o for o in c.encode("utf-8")])
break
return retval
def iri2uri(uri):
"""Convert an IRI to a URI. Note that IRIs must be
passed in a unicode strings. That is, do not utf-8 encode
the IRI before passing it into the function."""
if isinstance(uri, str):
(scheme, authority, path, query, fragment) = urllib.parse.urlsplit(uri)
authority = authority.encode("idna").decode("utf-8")
# For each character in 'ucschar' or 'iprivate'
# 1. encode as utf-8
# 2. then %-encode each octet of that utf-8
uri = urllib.parse.urlunsplit((scheme, authority, path, query, fragment))
uri = "".join([encode(c) for c in uri])
return uri
if __name__ == "__main__":
import unittest
class Test(unittest.TestCase):
def test_uris(self):
"""Test that URIs are invariant under the transformation."""
invariant = [
"ftp://ftp.is.co.za/rfc/rfc1808.txt",
"http://www.ietf.org/rfc/rfc2396.txt",
"ldap://[2001:db8::7]/c=GB?objectClass?one",
"mailto:John.Doe@example.com",
"news:comp.infosystems.www.servers.unix",
"tel:+1-816-555-1212",
"telnet://192.0.2.16:80/",
"urn:oasis:names:specification:docbook:dtd:xml:4.1.2",
]
for uri in invariant:
self.assertEqual(uri, iri2uri(uri))
def test_iri(self):
"""Test that the right type of escaping is done for each part of the URI."""
self.assertEqual(
"http://xn--o3h.com/%E2%98%84",
iri2uri("http://\N{COMET}.com/\N{COMET}"),
)
self.assertEqual(
"http://bitworking.org/?fred=%E2%98%84",
iri2uri("http://bitworking.org/?fred=\N{COMET}"),
)
self.assertEqual(
"http://bitworking.org/#%E2%98%84",
iri2uri("http://bitworking.org/#\N{COMET}"),
)
self.assertEqual("#%E2%98%84", iri2uri("#\N{COMET}"))
self.assertEqual(
"/fred?bar=%E2%98%9A#%E2%98%84",
iri2uri("/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"),
)
self.assertEqual(
"/fred?bar=%E2%98%9A#%E2%98%84",
iri2uri(iri2uri("/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")),
)
self.assertNotEqual(
"/fred?bar=%E2%98%9A#%E2%98%84",
iri2uri(
"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode("utf-8")
),
)
unittest.main()
lib/httplib2/py3/socks.py
+518
View File
@@ -0,0 +1,518 @@
"""SocksiPy - Python SOCKS module.
Version 1.00
Copyright 2006 Dan-Haim. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of Dan Haim nor the names of his contributors may be used
to endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
This module provides a standard socket-like interface for Python
for tunneling connections through SOCKS proxies.
Minor modifications made by Christopher Gilbert (http://motomastyle.com/) for
use in PyLoris (http://pyloris.sourceforge.net/).
Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/)
mainly to merge bug fixes found in Sourceforge.
"""
import base64
import socket
import struct
import sys
if getattr(socket, "socket", None) is None:
raise ImportError("socket.socket missing, proxy support unusable")
PROXY_TYPE_SOCKS4 = 1
PROXY_TYPE_SOCKS5 = 2
PROXY_TYPE_HTTP = 3
PROXY_TYPE_HTTP_NO_TUNNEL = 4
_defaultproxy = None
_orgsocket = socket.socket
class ProxyError(Exception):
pass
class GeneralProxyError(ProxyError):
pass
class Socks5AuthError(ProxyError):
pass
class Socks5Error(ProxyError):
pass
class Socks4Error(ProxyError):
pass
class HTTPError(ProxyError):
pass
_generalerrors = (
"success",
"invalid data",
"not connected",
"not available",
"bad proxy type",
"bad input",
)
_socks5errors = (
"succeeded",
"general SOCKS server failure",
"connection not allowed by ruleset",
"Network unreachable",
"Host unreachable",
"Connection refused",
"TTL expired",
"Command not supported",
"Address type not supported",
"Unknown error",
)
_socks5autherrors = (
"succeeded",
"authentication is required",
"all offered authentication methods were rejected",
"unknown username or invalid password",
"unknown error",
)
_socks4errors = (
"request granted",
"request rejected or failed",
"request rejected because SOCKS server cannot connect to identd on the client",
"request rejected because the client program and identd report different "
"user-ids",
"unknown error",
)
def setdefaultproxy(
proxytype=None, addr=None, port=None, rdns=True, username=None, password=None
):
"""setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
Sets a default proxy which all further socksocket objects will use,
unless explicitly changed.
"""
global _defaultproxy
_defaultproxy = (proxytype, addr, port, rdns, username, password)
def wrapmodule(module):
"""wrapmodule(module)
Attempts to replace a module's socket library with a SOCKS socket. Must set
a default proxy using setdefaultproxy(...) first.
This will only work on modules that import socket directly into the
namespace;
most of the Python Standard Library falls into this category.
"""
if _defaultproxy != None:
module.socket.socket = socksocket
else:
raise GeneralProxyError((4, "no proxy specified"))
class socksocket(socket.socket):
"""socksocket([family[, type[, proto]]]) -> socket object
Open a SOCKS enabled socket. The parameters are the same as
those of the standard socket init. In order for SOCKS to work,
you must specify family=AF_INET, type=SOCK_STREAM and proto=0.
"""
def __init__(
self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None
):
_orgsocket.__init__(self, family, type, proto, _sock)
if _defaultproxy != None:
self.__proxy = _defaultproxy
else:
self.__proxy = (None, None, None, None, None, None)
self.__proxysockname = None
self.__proxypeername = None
self.__httptunnel = True
def __recvall(self, count):
"""__recvall(count) -> data
Receive EXACTLY the number of bytes requested from the socket.
Blocks until the required number of bytes have been received.
"""
data = self.recv(count)
while len(data) < count:
d = self.recv(count - len(data))
if not d:
raise GeneralProxyError((0, "connection closed unexpectedly"))
data = data + d
return data
def sendall(self, content, *args):
""" override socket.socket.sendall method to rewrite the header
for non-tunneling proxies if needed
"""
if not self.__httptunnel:
content = self.__rewriteproxy(content)
return super(socksocket, self).sendall(content, *args)
def __rewriteproxy(self, header):
""" rewrite HTTP request headers to support non-tunneling proxies
(i.e. those which do not support the CONNECT method).
This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
"""
host, endpt = None, None
hdrs = header.split("\r\n")
for hdr in hdrs:
if hdr.lower().startswith("host:"):
host = hdr
elif hdr.lower().startswith("get") or hdr.lower().startswith("post"):
endpt = hdr
if host and endpt:
hdrs.remove(host)
hdrs.remove(endpt)
host = host.split(" ")[1]
endpt = endpt.split(" ")
if self.__proxy[4] != None and self.__proxy[5] != None:
hdrs.insert(0, self.__getauthheader())
hdrs.insert(0, "Host: %s" % host)
hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
return "\r\n".join(hdrs)
def __getauthheader(self):
auth = self.__proxy[4] + b":" + self.__proxy[5]
return "Proxy-Authorization: Basic " + base64.b64encode(auth).decode()
def setproxy(
self,
proxytype=None,
addr=None,
port=None,
rdns=True,
username=None,
password=None,
headers=None,
):
"""setproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
Sets the proxy to be used.
proxytype - The type of the proxy to be used. Three types
are supported: PROXY_TYPE_SOCKS4 (including socks4a),
PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
addr - The address of the server (IP or DNS).
port - The port of the server. Defaults to 1080 for SOCKS
servers and 8080 for HTTP proxy servers.
rdns - Should DNS queries be preformed on the remote side
(rather than the local side). The default is True.
Note: This has no effect with SOCKS4 servers.
username - Username to authenticate with to the server.
The default is no authentication.
password - Password to authenticate with to the server.
Only relevant when username is also provided.
headers - Additional or modified headers for the proxy connect
request.
"""
self.__proxy = (
proxytype,
addr,
port,
rdns,
username.encode() if username else None,
password.encode() if password else None,
headers,
)
def __negotiatesocks5(self, destaddr, destport):
"""__negotiatesocks5(self,destaddr,destport)
Negotiates a connection through a SOCKS5 server.
"""
# First we'll send the authentication packages we support.
if (self.__proxy[4] != None) and (self.__proxy[5] != None):
# The username/password details were supplied to the
# setproxy method so we support the USERNAME/PASSWORD
# authentication (in addition to the standard none).
self.sendall(struct.pack("BBBB", 0x05, 0x02, 0x00, 0x02))
else:
# No username/password were entered, therefore we
# only support connections with no authentication.
self.sendall(struct.pack("BBB", 0x05, 0x01, 0x00))
# We'll receive the server's response to determine which
# method was selected
chosenauth = self.__recvall(2)
if chosenauth[0:1] != chr(0x05).encode():
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
# Check the chosen authentication method
if chosenauth[1:2] == chr(0x00).encode():
# No authentication is required
pass
elif chosenauth[1:2] == chr(0x02).encode():
# Okay, we need to perform a basic username/password
# authentication.
packet = bytearray()
packet.append(0x01)
packet.append(len(self.__proxy[4]))
packet.extend(self.__proxy[4])
packet.append(len(self.__proxy[5]))
packet.extend(self.__proxy[5])
self.sendall(packet)
authstat = self.__recvall(2)
if authstat[0:1] != chr(0x01).encode():
# Bad response
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
if authstat[1:2] != chr(0x00).encode():
# Authentication failed
self.close()
raise Socks5AuthError((3, _socks5autherrors[3]))
# Authentication succeeded
else:
# Reaching here is always bad
self.close()
if chosenauth[1] == chr(0xFF).encode():
raise Socks5AuthError((2, _socks5autherrors[2]))
else:
raise GeneralProxyError((1, _generalerrors[1]))
# Now we can request the actual connection
req = struct.pack("BBB", 0x05, 0x01, 0x00)
# If the given destination address is an IP address, we'll
# use the IPv4 address request even if remote resolving was specified.
try:
ipaddr = socket.inet_aton(destaddr)
req = req + chr(0x01).encode() + ipaddr
except socket.error:
# Well it's not an IP number, so it's probably a DNS name.
if self.__proxy[3]:
# Resolve remotely
ipaddr = None
req = (
req
+ chr(0x03).encode()
+ chr(len(destaddr)).encode()
+ destaddr.encode()
)
else:
# Resolve locally
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
req = req + chr(0x01).encode() + ipaddr
req = req + struct.pack(">H", destport)
self.sendall(req)
# Get the response
resp = self.__recvall(4)
if resp[0:1] != chr(0x05).encode():
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
elif resp[1:2] != chr(0x00).encode():
# Connection failed
self.close()
if ord(resp[1:2]) <= 8:
raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
else:
raise Socks5Error((9, _socks5errors[9]))
# Get the bound address/port
elif resp[3:4] == chr(0x01).encode():
boundaddr = self.__recvall(4)
elif resp[3:4] == chr(0x03).encode():
resp = resp + self.recv(1)
boundaddr = self.__recvall(ord(resp[4:5]))
else:
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
boundport = struct.unpack(">H", self.__recvall(2))[0]
self.__proxysockname = (boundaddr, boundport)
if ipaddr != None:
self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
else:
self.__proxypeername = (destaddr, destport)
def getproxysockname(self):
"""getsockname() -> address info
Returns the bound IP address and port number at the proxy.
"""
return self.__proxysockname
def getproxypeername(self):
"""getproxypeername() -> address info
Returns the IP and port number of the proxy.
"""
return _orgsocket.getpeername(self)
def getpeername(self):
"""getpeername() -> address info
Returns the IP address and port number of the destination
machine (note: getproxypeername returns the proxy)
"""
return self.__proxypeername
def __negotiatesocks4(self, destaddr, destport):
"""__negotiatesocks4(self,destaddr,destport)
Negotiates a connection through a SOCKS4 server.
"""
# Check if the destination address provided is an IP address
rmtrslv = False
try:
ipaddr = socket.inet_aton(destaddr)
except socket.error:
# It's a DNS name. Check where it should be resolved.
if self.__proxy[3]:
ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01)
rmtrslv = True
else:
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
# Construct the request packet
req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr
# The username parameter is considered userid for SOCKS4
if self.__proxy[4] != None:
req = req + self.__proxy[4]
req = req + chr(0x00).encode()
# DNS name if remote resolving is required
# NOTE: This is actually an extension to the SOCKS4 protocol
# called SOCKS4A and may not be supported in all cases.
if rmtrslv:
req = req + destaddr + chr(0x00).encode()
self.sendall(req)
# Get the response from the server
resp = self.__recvall(8)
if resp[0:1] != chr(0x00).encode():
# Bad data
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
if resp[1:2] != chr(0x5A).encode():
# Server returned an error
self.close()
if ord(resp[1:2]) in (91, 92, 93):
self.close()
raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90]))
else:
raise Socks4Error((94, _socks4errors[4]))
# Get the bound address/port
self.__proxysockname = (
socket.inet_ntoa(resp[4:]),
struct.unpack(">H", resp[2:4])[0],
)
if rmtrslv != None:
self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
else:
self.__proxypeername = (destaddr, destport)
def __negotiatehttp(self, destaddr, destport):
"""__negotiatehttp(self,destaddr,destport)
Negotiates a connection through an HTTP server.
"""
# If we need to resolve locally, we do this now
if not self.__proxy[3]:
addr = socket.gethostbyname(destaddr)
else:
addr = destaddr
headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"]
wrote_host_header = False
wrote_auth_header = False
if self.__proxy[6] != None:
for key, val in self.__proxy[6].iteritems():
headers += [key, ": ", val, "\r\n"]
wrote_host_header = key.lower() == "host"
wrote_auth_header = key.lower() == "proxy-authorization"
if not wrote_host_header:
headers += ["Host: ", destaddr, "\r\n"]
if not wrote_auth_header:
if self.__proxy[4] != None and self.__proxy[5] != None:
headers += [self.__getauthheader(), "\r\n"]
headers.append("\r\n")
self.sendall("".join(headers).encode())
# We read the response until we get the string "\r\n\r\n"
resp = self.recv(1)
while resp.find("\r\n\r\n".encode()) == -1:
resp = resp + self.recv(1)
# We just need the first line to check if the connection
# was successful
statusline = resp.splitlines()[0].split(" ".encode(), 2)
if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()):
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
try:
statuscode = int(statusline[1])
except ValueError:
self.close()
raise GeneralProxyError((1, _generalerrors[1]))
if statuscode != 200:
self.close()
raise HTTPError((statuscode, statusline[2]))
self.__proxysockname = ("0.0.0.0", 0)
self.__proxypeername = (addr, destport)
def connect(self, destpair):
"""connect(self, despair)
Connects to the specified destination through a proxy.
destpar - A tuple of the IP/DNS address and the port number.
(identical to socket's connect).
To select the proxy server use setproxy().
"""
# Do a minimal input check first
if (
(not type(destpair) in (list, tuple))
or (len(destpair) < 2)
or (not isinstance(destpair[0], (str, bytes)))
or (type(destpair[1]) != int)
):
raise GeneralProxyError((5, _generalerrors[5]))
if self.__proxy[0] == PROXY_TYPE_SOCKS5:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 1080
_orgsocket.connect(self, (self.__proxy[1], portnum))
self.__negotiatesocks5(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_SOCKS4:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 1080
_orgsocket.connect(self, (self.__proxy[1], portnum))
self.__negotiatesocks4(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_HTTP:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 8080
_orgsocket.connect(self, (self.__proxy[1], portnum))
self.__negotiatehttp(destpair[0], destpair[1])
elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL:
if self.__proxy[2] != None:
portnum = self.__proxy[2]
else:
portnum = 8080
_orgsocket.connect(self, (self.__proxy[1], portnum))
if destpair[1] == 443:
self.__negotiatehttp(destpair[0], destpair[1])
else:
self.__httptunnel = False
elif self.__proxy[0] == None:
_orgsocket.connect(self, (destpair[0], destpair[1]))
else:
raise GeneralProxyError((4, _generalerrors[4]))
lib/reprlib/__init__.py
+9
View File
@@ -0,0 +1,9 @@
from __future__ import absolute_import
import sys
if sys.version_info[0] < 3:
from repr import *
else:
raise ImportError('This package should not be accessible on Python 3. '
'Either you are trying to run from the python-future src folder '
'or your installation of python-future is corrupted.')