Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

connection.py 33KB

1 year ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899
  1. from __future__ import annotations
  2. import datetime
  3. import logging
  4. import os
  5. import re
  6. import socket
  7. import typing
  8. import warnings
  9. from http.client import HTTPConnection as _HTTPConnection
  10. from http.client import HTTPException as HTTPException # noqa: F401
  11. from http.client import ResponseNotReady
  12. from socket import timeout as SocketTimeout
  13. if typing.TYPE_CHECKING:
  14. from typing_extensions import Literal
  15. from .response import HTTPResponse
  16. from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
  17. from .util.ssltransport import SSLTransport
  18. from ._collections import HTTPHeaderDict
  19. from .util.response import assert_header_parsing
  20. from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
  21. from .util.util import to_str
  22. from .util.wait import wait_for_read
  23. try: # Compiled with SSL?
  24. import ssl
  25. BaseSSLError = ssl.SSLError
  26. except (ImportError, AttributeError):
  27. ssl = None # type: ignore[assignment]
  28. class BaseSSLError(BaseException): # type: ignore[no-redef]
  29. pass
  30. from ._base_connection import _TYPE_BODY
  31. from ._base_connection import ProxyConfig as ProxyConfig
  32. from ._base_connection import _ResponseOptions as _ResponseOptions
  33. from ._version import __version__
  34. from .exceptions import (
  35. ConnectTimeoutError,
  36. HeaderParsingError,
  37. NameResolutionError,
  38. NewConnectionError,
  39. ProxyError,
  40. SystemTimeWarning,
  41. )
  42. from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
  43. from .util.request import body_to_chunks
  44. from .util.ssl_ import assert_fingerprint as _assert_fingerprint
  45. from .util.ssl_ import (
  46. create_urllib3_context,
  47. is_ipaddress,
  48. resolve_cert_reqs,
  49. resolve_ssl_version,
  50. ssl_wrap_socket,
  51. )
  52. from .util.ssl_match_hostname import CertificateError, match_hostname
  53. from .util.url import Url
  54. # Not a no-op, we're adding this to the namespace so it can be imported.
  55. ConnectionError = ConnectionError
  56. BrokenPipeError = BrokenPipeError
  57. log = logging.getLogger(__name__)
  58. port_by_scheme = {"http": 80, "https": 443}
  59. # When it comes time to update this value as a part of regular maintenance
  60. # (ie test_recent_date is failing) update it to ~6 months before the current date.
  61. RECENT_DATE = datetime.date(2022, 1, 1)
  62. _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
  63. class HTTPConnection(_HTTPConnection):
  64. """
  65. Based on :class:`http.client.HTTPConnection` but provides an extra constructor
  66. backwards-compatibility layer between older and newer Pythons.
  67. Additional keyword parameters are used to configure attributes of the connection.
  68. Accepted parameters include:
  69. - ``source_address``: Set the source address for the current connection.
  70. - ``socket_options``: Set specific options on the underlying socket. If not specified, then
  71. defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
  72. Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
  73. For example, if you wish to enable TCP Keep Alive in addition to the defaults,
  74. you might pass:
  75. .. code-block:: python
  76. HTTPConnection.default_socket_options + [
  77. (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
  78. ]
  79. Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
  80. """
  81. default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
  82. #: Disable Nagle's algorithm by default.
  83. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
  84. default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [
  85. (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
  86. ]
  87. #: Whether this connection verifies the host's certificate.
  88. is_verified: bool = False
  89. #: Whether this proxy connection verified the proxy host's certificate.
  90. # If no proxy is currently connected to the value will be ``None``.
  91. proxy_is_verified: bool | None = None
  92. blocksize: int
  93. source_address: tuple[str, int] | None
  94. socket_options: connection._TYPE_SOCKET_OPTIONS | None
  95. _has_connected_to_proxy: bool
  96. _response_options: _ResponseOptions | None
  97. _tunnel_host: str | None
  98. _tunnel_port: int | None
  99. _tunnel_scheme: str | None
  100. def __init__(
  101. self,
  102. host: str,
  103. port: int | None = None,
  104. *,
  105. timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
  106. source_address: tuple[str, int] | None = None,
  107. blocksize: int = 8192,
  108. socket_options: None
  109. | (connection._TYPE_SOCKET_OPTIONS) = default_socket_options,
  110. proxy: Url | None = None,
  111. proxy_config: ProxyConfig | None = None,
  112. ) -> None:
  113. super().__init__(
  114. host=host,
  115. port=port,
  116. timeout=Timeout.resolve_default_timeout(timeout),
  117. source_address=source_address,
  118. blocksize=blocksize,
  119. )
  120. self.socket_options = socket_options
  121. self.proxy = proxy
  122. self.proxy_config = proxy_config
  123. self._has_connected_to_proxy = False
  124. self._response_options = None
  125. self._tunnel_host: str | None = None
  126. self._tunnel_port: int | None = None
  127. self._tunnel_scheme: str | None = None
  128. # https://github.com/python/mypy/issues/4125
  129. # Mypy treats this as LSP violation, which is considered a bug.
  130. # If `host` is made a property it violates LSP, because a writeable attribute is overridden with a read-only one.
  131. # However, there is also a `host` setter so LSP is not violated.
  132. # Potentially, a `@host.deleter` might be needed depending on how this issue will be fixed.
  133. @property
  134. def host(self) -> str:
  135. """
  136. Getter method to remove any trailing dots that indicate the hostname is an FQDN.
  137. In general, SSL certificates don't include the trailing dot indicating a
  138. fully-qualified domain name, and thus, they don't validate properly when
  139. checked against a domain name that includes the dot. In addition, some
  140. servers may not expect to receive the trailing dot when provided.
  141. However, the hostname with trailing dot is critical to DNS resolution; doing a
  142. lookup with the trailing dot will properly only resolve the appropriate FQDN,
  143. whereas a lookup without a trailing dot will search the system's search domain
  144. list. Thus, it's important to keep the original host around for use only in
  145. those cases where it's appropriate (i.e., when doing DNS lookup to establish the
  146. actual TCP connection across which we're going to send HTTP requests).
  147. """
  148. return self._dns_host.rstrip(".")
  149. @host.setter
  150. def host(self, value: str) -> None:
  151. """
  152. Setter for the `host` property.
  153. We assume that only urllib3 uses the _dns_host attribute; httplib itself
  154. only uses `host`, and it seems reasonable that other libraries follow suit.
  155. """
  156. self._dns_host = value
  157. def _new_conn(self) -> socket.socket:
  158. """Establish a socket connection and set nodelay settings on it.
  159. :return: New socket connection.
  160. """
  161. try:
  162. sock = connection.create_connection(
  163. (self._dns_host, self.port),
  164. self.timeout,
  165. source_address=self.source_address,
  166. socket_options=self.socket_options,
  167. )
  168. except socket.gaierror as e:
  169. raise NameResolutionError(self.host, self, e) from e
  170. except SocketTimeout as e:
  171. raise ConnectTimeoutError(
  172. self,
  173. f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
  174. ) from e
  175. except OSError as e:
  176. raise NewConnectionError(
  177. self, f"Failed to establish a new connection: {e}"
  178. ) from e
  179. return sock
  180. def set_tunnel(
  181. self,
  182. host: str,
  183. port: int | None = None,
  184. headers: typing.Mapping[str, str] | None = None,
  185. scheme: str = "http",
  186. ) -> None:
  187. if scheme not in ("http", "https"):
  188. raise ValueError(
  189. f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
  190. )
  191. super().set_tunnel(host, port=port, headers=headers)
  192. self._tunnel_scheme = scheme
  193. def connect(self) -> None:
  194. self.sock = self._new_conn()
  195. if self._tunnel_host:
  196. # If we're tunneling it means we're connected to our proxy.
  197. self._has_connected_to_proxy = True
  198. # TODO: Fix tunnel so it doesn't depend on self.sock state.
  199. self._tunnel() # type: ignore[attr-defined]
  200. # If there's a proxy to be connected to we are fully connected.
  201. # This is set twice (once above and here) due to forwarding proxies
  202. # not using tunnelling.
  203. self._has_connected_to_proxy = bool(self.proxy)
  204. @property
  205. def is_closed(self) -> bool:
  206. return self.sock is None
  207. @property
  208. def is_connected(self) -> bool:
  209. if self.sock is None:
  210. return False
  211. return not wait_for_read(self.sock, timeout=0.0)
  212. @property
  213. def has_connected_to_proxy(self) -> bool:
  214. return self._has_connected_to_proxy
  215. def close(self) -> None:
  216. try:
  217. super().close()
  218. finally:
  219. # Reset all stateful properties so connection
  220. # can be re-used without leaking prior configs.
  221. self.sock = None
  222. self.is_verified = False
  223. self.proxy_is_verified = None
  224. self._has_connected_to_proxy = False
  225. self._response_options = None
  226. self._tunnel_host = None
  227. self._tunnel_port = None
  228. self._tunnel_scheme = None
  229. def putrequest(
  230. self,
  231. method: str,
  232. url: str,
  233. skip_host: bool = False,
  234. skip_accept_encoding: bool = False,
  235. ) -> None:
  236. """"""
  237. # Empty docstring because the indentation of CPython's implementation
  238. # is broken but we don't want this method in our documentation.
  239. match = _CONTAINS_CONTROL_CHAR_RE.search(method)
  240. if match:
  241. raise ValueError(
  242. f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
  243. )
  244. return super().putrequest(
  245. method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
  246. )
  247. def putheader(self, header: str, *values: str) -> None:
  248. """"""
  249. if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
  250. super().putheader(header, *values)
  251. elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
  252. skippable_headers = "', '".join(
  253. [str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
  254. )
  255. raise ValueError(
  256. f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
  257. )
  258. # `request` method's signature intentionally violates LSP.
  259. # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
  260. def request( # type: ignore[override]
  261. self,
  262. method: str,
  263. url: str,
  264. body: _TYPE_BODY | None = None,
  265. headers: typing.Mapping[str, str] | None = None,
  266. *,
  267. chunked: bool = False,
  268. preload_content: bool = True,
  269. decode_content: bool = True,
  270. enforce_content_length: bool = True,
  271. ) -> None:
  272. # Update the inner socket's timeout value to send the request.
  273. # This only triggers if the connection is re-used.
  274. if self.sock is not None:
  275. self.sock.settimeout(self.timeout)
  276. # Store these values to be fed into the HTTPResponse
  277. # object later. TODO: Remove this in favor of a real
  278. # HTTP lifecycle mechanism.
  279. # We have to store these before we call .request()
  280. # because sometimes we can still salvage a response
  281. # off the wire even if we aren't able to completely
  282. # send the request body.
  283. self._response_options = _ResponseOptions(
  284. request_method=method,
  285. request_url=url,
  286. preload_content=preload_content,
  287. decode_content=decode_content,
  288. enforce_content_length=enforce_content_length,
  289. )
  290. if headers is None:
  291. headers = {}
  292. header_keys = frozenset(to_str(k.lower()) for k in headers)
  293. skip_accept_encoding = "accept-encoding" in header_keys
  294. skip_host = "host" in header_keys
  295. self.putrequest(
  296. method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
  297. )
  298. # Transform the body into an iterable of sendall()-able chunks
  299. # and detect if an explicit Content-Length is doable.
  300. chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
  301. chunks = chunks_and_cl.chunks
  302. content_length = chunks_and_cl.content_length
  303. # When chunked is explicit set to 'True' we respect that.
  304. if chunked:
  305. if "transfer-encoding" not in header_keys:
  306. self.putheader("Transfer-Encoding", "chunked")
  307. else:
  308. # Detect whether a framing mechanism is already in use. If so
  309. # we respect that value, otherwise we pick chunked vs content-length
  310. # depending on the type of 'body'.
  311. if "content-length" in header_keys:
  312. chunked = False
  313. elif "transfer-encoding" in header_keys:
  314. chunked = True
  315. # Otherwise we go off the recommendation of 'body_to_chunks()'.
  316. else:
  317. chunked = False
  318. if content_length is None:
  319. if chunks is not None:
  320. chunked = True
  321. self.putheader("Transfer-Encoding", "chunked")
  322. else:
  323. self.putheader("Content-Length", str(content_length))
  324. # Now that framing headers are out of the way we send all the other headers.
  325. if "user-agent" not in header_keys:
  326. self.putheader("User-Agent", _get_default_user_agent())
  327. for header, value in headers.items():
  328. self.putheader(header, value)
  329. self.endheaders()
  330. # If we're given a body we start sending that in chunks.
  331. if chunks is not None:
  332. for chunk in chunks:
  333. # Sending empty chunks isn't allowed for TE: chunked
  334. # as it indicates the end of the body.
  335. if not chunk:
  336. continue
  337. if isinstance(chunk, str):
  338. chunk = chunk.encode("utf-8")
  339. if chunked:
  340. self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
  341. else:
  342. self.send(chunk)
  343. # Regardless of whether we have a body or not, if we're in
  344. # chunked mode we want to send an explicit empty chunk.
  345. if chunked:
  346. self.send(b"0\r\n\r\n")
  347. def request_chunked(
  348. self,
  349. method: str,
  350. url: str,
  351. body: _TYPE_BODY | None = None,
  352. headers: typing.Mapping[str, str] | None = None,
  353. ) -> None:
  354. """
  355. Alternative to the common request method, which sends the
  356. body with chunked encoding and not as one block
  357. """
  358. warnings.warn(
  359. "HTTPConnection.request_chunked() is deprecated and will be removed "
  360. "in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).",
  361. category=DeprecationWarning,
  362. stacklevel=2,
  363. )
  364. self.request(method, url, body=body, headers=headers, chunked=True)
  365. def getresponse( # type: ignore[override]
  366. self,
  367. ) -> HTTPResponse:
  368. """
  369. Get the response from the server.
  370. If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
  371. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
  372. """
  373. # Raise the same error as http.client.HTTPConnection
  374. if self._response_options is None:
  375. raise ResponseNotReady()
  376. # Reset this attribute for being used again.
  377. resp_options = self._response_options
  378. self._response_options = None
  379. # Since the connection's timeout value may have been updated
  380. # we need to set the timeout on the socket.
  381. self.sock.settimeout(self.timeout)
  382. # This is needed here to avoid circular import errors
  383. from .response import HTTPResponse
  384. # Get the response from http.client.HTTPConnection
  385. httplib_response = super().getresponse()
  386. try:
  387. assert_header_parsing(httplib_response.msg)
  388. except (HeaderParsingError, TypeError) as hpe:
  389. log.warning(
  390. "Failed to parse headers (url=%s): %s",
  391. _url_from_connection(self, resp_options.request_url),
  392. hpe,
  393. exc_info=True,
  394. )
  395. headers = HTTPHeaderDict(httplib_response.msg.items())
  396. response = HTTPResponse(
  397. body=httplib_response,
  398. headers=headers,
  399. status=httplib_response.status,
  400. version=httplib_response.version,
  401. reason=httplib_response.reason,
  402. preload_content=resp_options.preload_content,
  403. decode_content=resp_options.decode_content,
  404. original_response=httplib_response,
  405. enforce_content_length=resp_options.enforce_content_length,
  406. request_method=resp_options.request_method,
  407. request_url=resp_options.request_url,
  408. )
  409. return response
  410. class HTTPSConnection(HTTPConnection):
  411. """
  412. Many of the parameters to this constructor are passed to the underlying SSL
  413. socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
  414. """
  415. default_port = port_by_scheme["https"] # type: ignore[misc]
  416. cert_reqs: int | str | None = None
  417. ca_certs: str | None = None
  418. ca_cert_dir: str | None = None
  419. ca_cert_data: None | str | bytes = None
  420. ssl_version: int | str | None = None
  421. ssl_minimum_version: int | None = None
  422. ssl_maximum_version: int | None = None
  423. assert_fingerprint: str | None = None
  424. def __init__(
  425. self,
  426. host: str,
  427. port: int | None = None,
  428. *,
  429. timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
  430. source_address: tuple[str, int] | None = None,
  431. blocksize: int = 8192,
  432. socket_options: None
  433. | (connection._TYPE_SOCKET_OPTIONS) = HTTPConnection.default_socket_options,
  434. proxy: Url | None = None,
  435. proxy_config: ProxyConfig | None = None,
  436. cert_reqs: int | str | None = None,
  437. assert_hostname: None | str | Literal[False] = None,
  438. assert_fingerprint: str | None = None,
  439. server_hostname: str | None = None,
  440. ssl_context: ssl.SSLContext | None = None,
  441. ca_certs: str | None = None,
  442. ca_cert_dir: str | None = None,
  443. ca_cert_data: None | str | bytes = None,
  444. ssl_minimum_version: int | None = None,
  445. ssl_maximum_version: int | None = None,
  446. ssl_version: int | str | None = None, # Deprecated
  447. cert_file: str | None = None,
  448. key_file: str | None = None,
  449. key_password: str | None = None,
  450. ) -> None:
  451. super().__init__(
  452. host,
  453. port=port,
  454. timeout=timeout,
  455. source_address=source_address,
  456. blocksize=blocksize,
  457. socket_options=socket_options,
  458. proxy=proxy,
  459. proxy_config=proxy_config,
  460. )
  461. self.key_file = key_file
  462. self.cert_file = cert_file
  463. self.key_password = key_password
  464. self.ssl_context = ssl_context
  465. self.server_hostname = server_hostname
  466. self.assert_hostname = assert_hostname
  467. self.assert_fingerprint = assert_fingerprint
  468. self.ssl_version = ssl_version
  469. self.ssl_minimum_version = ssl_minimum_version
  470. self.ssl_maximum_version = ssl_maximum_version
  471. self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
  472. self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
  473. self.ca_cert_data = ca_cert_data
  474. # cert_reqs depends on ssl_context so calculate last.
  475. if cert_reqs is None:
  476. if self.ssl_context is not None:
  477. cert_reqs = self.ssl_context.verify_mode
  478. else:
  479. cert_reqs = resolve_cert_reqs(None)
  480. self.cert_reqs = cert_reqs
  481. def set_cert(
  482. self,
  483. key_file: str | None = None,
  484. cert_file: str | None = None,
  485. cert_reqs: int | str | None = None,
  486. key_password: str | None = None,
  487. ca_certs: str | None = None,
  488. assert_hostname: None | str | Literal[False] = None,
  489. assert_fingerprint: str | None = None,
  490. ca_cert_dir: str | None = None,
  491. ca_cert_data: None | str | bytes = None,
  492. ) -> None:
  493. """
  494. This method should only be called once, before the connection is used.
  495. """
  496. warnings.warn(
  497. "HTTPSConnection.set_cert() is deprecated and will be removed "
  498. "in urllib3 v2.1.0. Instead provide the parameters to the "
  499. "HTTPSConnection constructor.",
  500. category=DeprecationWarning,
  501. stacklevel=2,
  502. )
  503. # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
  504. # have an SSLContext object in which case we'll use its verify_mode.
  505. if cert_reqs is None:
  506. if self.ssl_context is not None:
  507. cert_reqs = self.ssl_context.verify_mode
  508. else:
  509. cert_reqs = resolve_cert_reqs(None)
  510. self.key_file = key_file
  511. self.cert_file = cert_file
  512. self.cert_reqs = cert_reqs
  513. self.key_password = key_password
  514. self.assert_hostname = assert_hostname
  515. self.assert_fingerprint = assert_fingerprint
  516. self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
  517. self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
  518. self.ca_cert_data = ca_cert_data
  519. def connect(self) -> None:
  520. sock: socket.socket | ssl.SSLSocket
  521. self.sock = sock = self._new_conn()
  522. server_hostname: str = self.host
  523. tls_in_tls = False
  524. # Do we need to establish a tunnel?
  525. if self._tunnel_host is not None:
  526. # We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
  527. if self._tunnel_scheme == "https":
  528. self.sock = sock = self._connect_tls_proxy(self.host, sock)
  529. tls_in_tls = True
  530. # If we're tunneling it means we're connected to our proxy.
  531. self._has_connected_to_proxy = True
  532. self._tunnel() # type: ignore[attr-defined]
  533. # Override the host with the one we're requesting data from.
  534. server_hostname = self._tunnel_host
  535. if self.server_hostname is not None:
  536. server_hostname = self.server_hostname
  537. is_time_off = datetime.date.today() < RECENT_DATE
  538. if is_time_off:
  539. warnings.warn(
  540. (
  541. f"System time is way off (before {RECENT_DATE}). This will probably "
  542. "lead to SSL verification errors"
  543. ),
  544. SystemTimeWarning,
  545. )
  546. sock_and_verified = _ssl_wrap_socket_and_match_hostname(
  547. sock=sock,
  548. cert_reqs=self.cert_reqs,
  549. ssl_version=self.ssl_version,
  550. ssl_minimum_version=self.ssl_minimum_version,
  551. ssl_maximum_version=self.ssl_maximum_version,
  552. ca_certs=self.ca_certs,
  553. ca_cert_dir=self.ca_cert_dir,
  554. ca_cert_data=self.ca_cert_data,
  555. cert_file=self.cert_file,
  556. key_file=self.key_file,
  557. key_password=self.key_password,
  558. server_hostname=server_hostname,
  559. ssl_context=self.ssl_context,
  560. tls_in_tls=tls_in_tls,
  561. assert_hostname=self.assert_hostname,
  562. assert_fingerprint=self.assert_fingerprint,
  563. )
  564. self.sock = sock_and_verified.socket
  565. self.is_verified = sock_and_verified.is_verified
  566. # If there's a proxy to be connected to we are fully connected.
  567. # This is set twice (once above and here) due to forwarding proxies
  568. # not using tunnelling.
  569. self._has_connected_to_proxy = bool(self.proxy)
  570. def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
  571. """
  572. Establish a TLS connection to the proxy using the provided SSL context.
  573. """
  574. # `_connect_tls_proxy` is called when self._tunnel_host is truthy.
  575. proxy_config = typing.cast(ProxyConfig, self.proxy_config)
  576. ssl_context = proxy_config.ssl_context
  577. sock_and_verified = _ssl_wrap_socket_and_match_hostname(
  578. sock,
  579. cert_reqs=self.cert_reqs,
  580. ssl_version=self.ssl_version,
  581. ssl_minimum_version=self.ssl_minimum_version,
  582. ssl_maximum_version=self.ssl_maximum_version,
  583. ca_certs=self.ca_certs,
  584. ca_cert_dir=self.ca_cert_dir,
  585. ca_cert_data=self.ca_cert_data,
  586. server_hostname=hostname,
  587. ssl_context=ssl_context,
  588. assert_hostname=proxy_config.assert_hostname,
  589. assert_fingerprint=proxy_config.assert_fingerprint,
  590. # Features that aren't implemented for proxies yet:
  591. cert_file=None,
  592. key_file=None,
  593. key_password=None,
  594. tls_in_tls=False,
  595. )
  596. self.proxy_is_verified = sock_and_verified.is_verified
  597. return sock_and_verified.socket # type: ignore[return-value]
  598. class _WrappedAndVerifiedSocket(typing.NamedTuple):
  599. """
  600. Wrapped socket and whether the connection is
  601. verified after the TLS handshake
  602. """
  603. socket: ssl.SSLSocket | SSLTransport
  604. is_verified: bool
  605. def _ssl_wrap_socket_and_match_hostname(
  606. sock: socket.socket,
  607. *,
  608. cert_reqs: None | str | int,
  609. ssl_version: None | str | int,
  610. ssl_minimum_version: int | None,
  611. ssl_maximum_version: int | None,
  612. cert_file: str | None,
  613. key_file: str | None,
  614. key_password: str | None,
  615. ca_certs: str | None,
  616. ca_cert_dir: str | None,
  617. ca_cert_data: None | str | bytes,
  618. assert_hostname: None | str | Literal[False],
  619. assert_fingerprint: str | None,
  620. server_hostname: str | None,
  621. ssl_context: ssl.SSLContext | None,
  622. tls_in_tls: bool = False,
  623. ) -> _WrappedAndVerifiedSocket:
  624. """Logic for constructing an SSLContext from all TLS parameters, passing
  625. that down into ssl_wrap_socket, and then doing certificate verification
  626. either via hostname or fingerprint. This function exists to guarantee
  627. that both proxies and targets have the same behavior when connecting via TLS.
  628. """
  629. default_ssl_context = False
  630. if ssl_context is None:
  631. default_ssl_context = True
  632. context = create_urllib3_context(
  633. ssl_version=resolve_ssl_version(ssl_version),
  634. ssl_minimum_version=ssl_minimum_version,
  635. ssl_maximum_version=ssl_maximum_version,
  636. cert_reqs=resolve_cert_reqs(cert_reqs),
  637. )
  638. else:
  639. context = ssl_context
  640. context.verify_mode = resolve_cert_reqs(cert_reqs)
  641. # In some cases, we want to verify hostnames ourselves
  642. if (
  643. # `ssl` can't verify fingerprints or alternate hostnames
  644. assert_fingerprint
  645. or assert_hostname
  646. # assert_hostname can be set to False to disable hostname checking
  647. or assert_hostname is False
  648. # We still support OpenSSL 1.0.2, which prevents us from verifying
  649. # hostnames easily: https://github.com/pyca/pyopenssl/pull/933
  650. or ssl_.IS_PYOPENSSL
  651. or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
  652. ):
  653. context.check_hostname = False
  654. # Try to load OS default certs if none are given.
  655. # We need to do the hasattr() check for our custom
  656. # pyOpenSSL and SecureTransport SSLContext objects
  657. # because neither support load_default_certs().
  658. if (
  659. not ca_certs
  660. and not ca_cert_dir
  661. and not ca_cert_data
  662. and default_ssl_context
  663. and hasattr(context, "load_default_certs")
  664. ):
  665. context.load_default_certs()
  666. # Ensure that IPv6 addresses are in the proper format and don't have a
  667. # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
  668. # and interprets them as DNS hostnames.
  669. if server_hostname is not None:
  670. normalized = server_hostname.strip("[]")
  671. if "%" in normalized:
  672. normalized = normalized[: normalized.rfind("%")]
  673. if is_ipaddress(normalized):
  674. server_hostname = normalized
  675. ssl_sock = ssl_wrap_socket(
  676. sock=sock,
  677. keyfile=key_file,
  678. certfile=cert_file,
  679. key_password=key_password,
  680. ca_certs=ca_certs,
  681. ca_cert_dir=ca_cert_dir,
  682. ca_cert_data=ca_cert_data,
  683. server_hostname=server_hostname,
  684. ssl_context=context,
  685. tls_in_tls=tls_in_tls,
  686. )
  687. try:
  688. if assert_fingerprint:
  689. _assert_fingerprint(
  690. ssl_sock.getpeercert(binary_form=True), assert_fingerprint
  691. )
  692. elif (
  693. context.verify_mode != ssl.CERT_NONE
  694. and not context.check_hostname
  695. and assert_hostname is not False
  696. ):
  697. cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
  698. # Need to signal to our match_hostname whether to use 'commonName' or not.
  699. # If we're using our own constructed SSLContext we explicitly set 'False'
  700. # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
  701. if default_ssl_context:
  702. hostname_checks_common_name = False
  703. else:
  704. hostname_checks_common_name = (
  705. getattr(context, "hostname_checks_common_name", False) or False
  706. )
  707. _match_hostname(
  708. cert,
  709. assert_hostname or server_hostname, # type: ignore[arg-type]
  710. hostname_checks_common_name,
  711. )
  712. return _WrappedAndVerifiedSocket(
  713. socket=ssl_sock,
  714. is_verified=context.verify_mode == ssl.CERT_REQUIRED
  715. or bool(assert_fingerprint),
  716. )
  717. except BaseException:
  718. ssl_sock.close()
  719. raise
  720. def _match_hostname(
  721. cert: _TYPE_PEER_CERT_RET_DICT | None,
  722. asserted_hostname: str,
  723. hostname_checks_common_name: bool = False,
  724. ) -> None:
  725. # Our upstream implementation of ssl.match_hostname()
  726. # only applies this normalization to IP addresses so it doesn't
  727. # match DNS SANs so we do the same thing!
  728. stripped_hostname = asserted_hostname.strip("[]")
  729. if is_ipaddress(stripped_hostname):
  730. asserted_hostname = stripped_hostname
  731. try:
  732. match_hostname(cert, asserted_hostname, hostname_checks_common_name)
  733. except CertificateError as e:
  734. log.warning(
  735. "Certificate did not match expected hostname: %s. Certificate: %s",
  736. asserted_hostname,
  737. cert,
  738. )
  739. # Add cert to exception and reraise so client code can inspect
  740. # the cert when catching the exception, if they want to
  741. e._peer_cert = cert # type: ignore[attr-defined]
  742. raise
  743. def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
  744. # Look for the phrase 'wrong version number', if found
  745. # then we should warn the user that we're very sure that
  746. # this proxy is HTTP-only and they have a configuration issue.
  747. error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
  748. is_likely_http_proxy = (
  749. "wrong version number" in error_normalized
  750. or "unknown protocol" in error_normalized
  751. )
  752. http_proxy_warning = (
  753. ". Your proxy appears to only use HTTP and not HTTPS, "
  754. "try changing your proxy URL to be HTTP. See: "
  755. "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
  756. "#https-proxy-error-http-proxy"
  757. )
  758. new_err = ProxyError(
  759. f"Unable to connect to proxy"
  760. f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
  761. err,
  762. )
  763. new_err.__cause__ = err
  764. return new_err
  765. def _get_default_user_agent() -> str:
  766. return f"python-urllib3/{__version__}"
  767. class DummyConnection:
  768. """Used to detect a failed ConnectionCls import."""
  769. if not ssl:
  770. HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
  771. VerifiedHTTPSConnection = HTTPSConnection
  772. def _url_from_connection(
  773. conn: HTTPConnection | HTTPSConnection, path: str | None = None
  774. ) -> str:
  775. """Returns the URL from a given connection. This is mainly used for testing and logging."""
  776. scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
  777. return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url