264 lines
9.7 KiB
Python
264 lines
9.7 KiB
Python
|
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
||
|
|
|
||
|
|
from requests import adapters
|
||
|
|
from requests.utils import (
|
||
|
|
DEFAULT_CA_BUNDLE_PATH,
|
||
|
|
get_auth_from_url,
|
||
|
|
get_encoding_from_headers,
|
||
|
|
prepend_scheme_if_needed,
|
||
|
|
select_proxy,
|
||
|
|
urldefragauth,
|
||
|
|
)
|
||
|
|
from urllib3.util import parse_url
|
||
|
|
from urllib3.util.retry import Retry
|
||
|
|
from urllib3.util import Timeout as TimeoutSauce
|
||
|
|
|
||
|
|
DEFAULT_POOLBLOCK = False
|
||
|
|
DEFAULT_POOLSIZE = 10
|
||
|
|
DEFAULT_RETRIES = 0
|
||
|
|
DEFAULT_POOL_TIMEOUT = None
|
||
|
|
|
||
|
|
class HTTPAdapter(adapters.HTTPAdapter):
|
||
|
|
def __init__(self,
|
||
|
|
pool_connections=DEFAULT_POOLSIZE,
|
||
|
|
pool_maxsize=DEFAULT_POOLSIZE,
|
||
|
|
max_retries=DEFAULT_RETRIES,
|
||
|
|
pool_block=DEFAULT_POOLBLOCK
|
||
|
|
):
|
||
|
|
self.config = {}
|
||
|
|
self.proxy_manager = {}
|
||
|
|
|
||
|
|
if isinstance(max_retries, Retry):
|
||
|
|
self.max_retries = max_retries
|
||
|
|
else:
|
||
|
|
max_retries = Retry.from_int(max_retries)
|
||
|
|
self.max_retries = max_retries
|
||
|
|
|
||
|
|
self._pool_connections = pool_connections
|
||
|
|
self._pool_maxsize = pool_maxsize
|
||
|
|
self._pool_block = pool_block
|
||
|
|
|
||
|
|
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
|
||
|
|
|
||
|
|
|
||
|
|
class HTTPSAdapter(HTTPAdapter):
|
||
|
|
"""The built-in HTTP Adapter for urllib3.
|
||
|
|
|
||
|
|
Provides a general-case interface for Requests sessions to contact HTTP and
|
||
|
|
HTTPS urls by implementing the Transport Adapter interface. This class will
|
||
|
|
usually be created by the :class:`Session <Session>` class under the
|
||
|
|
covers.
|
||
|
|
|
||
|
|
:param pool_connections: The number of urllib3 connection pools to cache.
|
||
|
|
:param pool_maxsize: The maximum number of connections to save in the pool.
|
||
|
|
:param max_retries: The maximum number of retries each connection
|
||
|
|
should attempt. Note, this applies only to failed DNS lookups, socket
|
||
|
|
connections and connection timeouts, never to requests where data has
|
||
|
|
made it to the server. By default, Requests does not retry failed
|
||
|
|
connections. If you need granular control over the conditions under
|
||
|
|
which we retry a request, import urllib3's ``Retry`` class and pass
|
||
|
|
that instead.
|
||
|
|
:param pool_block: Whether the connection pool should block for connections.
|
||
|
|
|
||
|
|
Usage::
|
||
|
|
|
||
|
|
>>> import requests
|
||
|
|
>>> s = requests.Session()
|
||
|
|
>>> a = requests.adapters.HTTPAdapter(max_retries=3)
|
||
|
|
>>> s.mount('http://', a)
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
pool_connections=DEFAULT_POOLSIZE,
|
||
|
|
pool_maxsize=1,
|
||
|
|
max_retries=3,
|
||
|
|
pool_block=DEFAULT_POOLBLOCK,
|
||
|
|
):
|
||
|
|
retries = Retry(connect=max_retries, read=2, redirect=0)
|
||
|
|
adapters.HTTPAdapter.__init__(self,
|
||
|
|
pool_connections=pool_connections,
|
||
|
|
pool_maxsize=pool_maxsize,
|
||
|
|
max_retries=retries,
|
||
|
|
pool_block=pool_block)
|
||
|
|
|
||
|
|
def get_connection(self, url, proxies=None, use_forwarding_for_https=True):
|
||
|
|
"""Returns a urllib3 connection for the given URL. This should not be
|
||
|
|
called from user code, and is only exposed for use when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param url: The URL to connect to.
|
||
|
|
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
|
||
|
|
:rtype: urllib3.ConnectionPool
|
||
|
|
"""
|
||
|
|
proxy = select_proxy(url, proxies)
|
||
|
|
|
||
|
|
if proxy:
|
||
|
|
proxy = prepend_scheme_if_needed(proxy, "http")
|
||
|
|
proxy_url = parse_url(proxy)
|
||
|
|
if not proxy_url.host:
|
||
|
|
raise InvalidProxyURL(
|
||
|
|
"Please check proxy URL. It is malformed "
|
||
|
|
"and could be missing the host."
|
||
|
|
)
|
||
|
|
proxy_manager = self.proxy_manager_for(proxy)
|
||
|
|
conn = proxy_manager.connection_from_url(url)
|
||
|
|
else:
|
||
|
|
# Only scheme should be lower case
|
||
|
|
parsed = urlparse(url)
|
||
|
|
url = parsed.geturl()
|
||
|
|
conn = self.poolmanager.connection_from_url(url, use_forwarding_for_https=True)
|
||
|
|
|
||
|
|
return conn
|
||
|
|
|
||
|
|
def send(
|
||
|
|
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
|
||
|
|
):
|
||
|
|
"""Sends PreparedRequest object. Returns Response object.
|
||
|
|
|
||
|
|
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||
|
|
:param stream: (optional) Whether to stream the request content.
|
||
|
|
:param timeout: (optional) How long to wait for the server to send
|
||
|
|
data before giving up, as a float, or a :ref:`(connect timeout,
|
||
|
|
read timeout) <timeouts>` tuple.
|
||
|
|
:type timeout: float or tuple or urllib3 Timeout object
|
||
|
|
:param verify: (optional) Either a boolean, in which case it controls whether
|
||
|
|
we verify the server's TLS certificate, or a string, in which case it
|
||
|
|
must be a path to a CA bundle to use
|
||
|
|
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||
|
|
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||
|
|
:rtype: requests.Response
|
||
|
|
"""
|
||
|
|
|
||
|
|
try:
|
||
|
|
#? _socks_options
|
||
|
|
conn = self.get_connection(request.url, proxies, use_forwarding_for_https=True)
|
||
|
|
except LocationValueError as e:
|
||
|
|
raise InvalidURL(e, request=request)
|
||
|
|
|
||
|
|
self.cert_verify(conn, request.url, verify, cert)
|
||
|
|
url = self.request_url(request, proxies)
|
||
|
|
self.add_headers(
|
||
|
|
request,
|
||
|
|
stream=stream,
|
||
|
|
timeout=timeout,
|
||
|
|
verify=verify,
|
||
|
|
cert=cert,
|
||
|
|
proxies=proxies,
|
||
|
|
)
|
||
|
|
|
||
|
|
chunked = not (request.body is None or "Content-Length" in request.headers)
|
||
|
|
|
||
|
|
if isinstance(timeout, tuple):
|
||
|
|
try:
|
||
|
|
connect, read = timeout
|
||
|
|
timeout = TimeoutSauce(connect=connect, read=read)
|
||
|
|
except ValueError:
|
||
|
|
raise ValueError(
|
||
|
|
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
|
||
|
|
f"or a single float to set both timeouts to the same value."
|
||
|
|
)
|
||
|
|
elif isinstance(timeout, TimeoutSauce):
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
timeout = TimeoutSauce(connect=timeout, read=timeout)
|
||
|
|
|
||
|
|
try:
|
||
|
|
if not chunked:
|
||
|
|
resp = conn.urlopen(
|
||
|
|
method=request.method,
|
||
|
|
url=url,
|
||
|
|
body=request.body,
|
||
|
|
headers=request.headers,
|
||
|
|
redirect=False,
|
||
|
|
assert_same_host=False,
|
||
|
|
preload_content=False,
|
||
|
|
decode_content=False,
|
||
|
|
retries=self.max_retries,
|
||
|
|
timeout=timeout,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Send the request.
|
||
|
|
else:
|
||
|
|
if hasattr(conn, "proxy_pool"):
|
||
|
|
conn = conn.proxy_pool
|
||
|
|
|
||
|
|
low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
|
||
|
|
|
||
|
|
try:
|
||
|
|
skip_host = "Host" in request.headers
|
||
|
|
low_conn.putrequest(
|
||
|
|
request.method,
|
||
|
|
url,
|
||
|
|
skip_accept_encoding=True,
|
||
|
|
skip_host=skip_host,
|
||
|
|
)
|
||
|
|
|
||
|
|
for header, value in request.headers.items():
|
||
|
|
low_conn.putheader(header, value)
|
||
|
|
|
||
|
|
low_conn.endheaders()
|
||
|
|
|
||
|
|
for i in request.body:
|
||
|
|
low_conn.send(hex(len(i))[2:].encode("utf-8"))
|
||
|
|
low_conn.send(b"\r\n")
|
||
|
|
low_conn.send(i)
|
||
|
|
low_conn.send(b"\r\n")
|
||
|
|
low_conn.send(b"0\r\n\r\n")
|
||
|
|
|
||
|
|
# Receive the response from the server
|
||
|
|
r = low_conn.getresponse()
|
||
|
|
|
||
|
|
resp = HTTPResponse.from_httplib(
|
||
|
|
r,
|
||
|
|
pool=conn,
|
||
|
|
connection=low_conn,
|
||
|
|
preload_content=False,
|
||
|
|
decode_content=False,
|
||
|
|
)
|
||
|
|
except Exception:
|
||
|
|
# If we hit any problems here, clean up the connection.
|
||
|
|
# Then, raise so that we can handle the actual exception.
|
||
|
|
low_conn.close()
|
||
|
|
raise
|
||
|
|
|
||
|
|
except (ProtocolError, OSError) as err:
|
||
|
|
raise ConnectionError(err, request=request)
|
||
|
|
|
||
|
|
except MaxRetryError as e:
|
||
|
|
if isinstance(e.reason, ConnectTimeoutError):
|
||
|
|
# TODO: Remove this in 3.0.0: see #2811
|
||
|
|
if not isinstance(e.reason, NewConnectionError):
|
||
|
|
raise ConnectTimeout(e, request=request)
|
||
|
|
|
||
|
|
if isinstance(e.reason, ResponseError):
|
||
|
|
raise RetryError(e, request=request)
|
||
|
|
|
||
|
|
if isinstance(e.reason, _ProxyError):
|
||
|
|
raise ProxyError(e, request=request)
|
||
|
|
|
||
|
|
if isinstance(e.reason, _SSLError):
|
||
|
|
# This branch is for urllib3 v1.22 and later.
|
||
|
|
raise SSLError(e, request=request)
|
||
|
|
|
||
|
|
raise ConnectionError(e, request=request)
|
||
|
|
|
||
|
|
except ClosedPoolError as e:
|
||
|
|
raise ConnectionError(e, request=request)
|
||
|
|
|
||
|
|
except _ProxyError as e:
|
||
|
|
raise ProxyError(e)
|
||
|
|
|
||
|
|
except (_SSLError, _HTTPError) as e:
|
||
|
|
if isinstance(e, _SSLError):
|
||
|
|
# This branch is for urllib3 versions earlier than v1.22
|
||
|
|
raise SSLError(e, request=request)
|
||
|
|
elif isinstance(e, ReadTimeoutError):
|
||
|
|
raise ReadTimeout(e, request=request)
|
||
|
|
elif isinstance(e, _InvalidHeader):
|
||
|
|
raise InvalidHeader(e, request=request)
|
||
|
|
else:
|
||
|
|
raise
|
||
|
|
|
||
|
|
return self.build_response(request, resp)
|