Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

poolmanager.py 22KB

1 year ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634
  1. from __future__ import annotations
  2. import functools
  3. import logging
  4. import typing
  5. import warnings
  6. from types import TracebackType
  7. from urllib.parse import urljoin
  8. from ._collections import RecentlyUsedContainer
  9. from ._request_methods import RequestMethods
  10. from .connection import ProxyConfig
  11. from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
  12. from .exceptions import (
  13. LocationValueError,
  14. MaxRetryError,
  15. ProxySchemeUnknown,
  16. URLSchemeUnknown,
  17. )
  18. from .response import BaseHTTPResponse
  19. from .util.connection import _TYPE_SOCKET_OPTIONS
  20. from .util.proxy import connection_requires_http_tunnel
  21. from .util.retry import Retry
  22. from .util.timeout import Timeout
  23. from .util.url import Url, parse_url
  24. if typing.TYPE_CHECKING:
  25. import ssl
  26. from typing_extensions import Literal
  27. __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
  28. log = logging.getLogger(__name__)
  29. SSL_KEYWORDS = (
  30. "key_file",
  31. "cert_file",
  32. "cert_reqs",
  33. "ca_certs",
  34. "ssl_version",
  35. "ssl_minimum_version",
  36. "ssl_maximum_version",
  37. "ca_cert_dir",
  38. "ssl_context",
  39. "key_password",
  40. "server_hostname",
  41. )
  42. # Default value for `blocksize` - a new parameter introduced to
  43. # http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
  44. _DEFAULT_BLOCKSIZE = 16384
  45. _SelfT = typing.TypeVar("_SelfT")
  46. class PoolKey(typing.NamedTuple):
  47. """
  48. All known keyword arguments that could be provided to the pool manager, its
  49. pools, or the underlying connections.
  50. All custom key schemes should include the fields in this key at a minimum.
  51. """
  52. key_scheme: str
  53. key_host: str
  54. key_port: int | None
  55. key_timeout: Timeout | float | int | None
  56. key_retries: Retry | bool | int | None
  57. key_block: bool | None
  58. key_source_address: tuple[str, int] | None
  59. key_key_file: str | None
  60. key_key_password: str | None
  61. key_cert_file: str | None
  62. key_cert_reqs: str | None
  63. key_ca_certs: str | None
  64. key_ssl_version: int | str | None
  65. key_ssl_minimum_version: ssl.TLSVersion | None
  66. key_ssl_maximum_version: ssl.TLSVersion | None
  67. key_ca_cert_dir: str | None
  68. key_ssl_context: ssl.SSLContext | None
  69. key_maxsize: int | None
  70. key_headers: frozenset[tuple[str, str]] | None
  71. key__proxy: Url | None
  72. key__proxy_headers: frozenset[tuple[str, str]] | None
  73. key__proxy_config: ProxyConfig | None
  74. key_socket_options: _TYPE_SOCKET_OPTIONS | None
  75. key__socks_options: frozenset[tuple[str, str]] | None
  76. key_assert_hostname: bool | str | None
  77. key_assert_fingerprint: str | None
  78. key_server_hostname: str | None
  79. key_blocksize: int | None
  80. def _default_key_normalizer(
  81. key_class: type[PoolKey], request_context: dict[str, typing.Any]
  82. ) -> PoolKey:
  83. """
  84. Create a pool key out of a request context dictionary.
  85. According to RFC 3986, both the scheme and host are case-insensitive.
  86. Therefore, this function normalizes both before constructing the pool
  87. key for an HTTPS request. If you wish to change this behaviour, provide
  88. alternate callables to ``key_fn_by_scheme``.
  89. :param key_class:
  90. The class to use when constructing the key. This should be a namedtuple
  91. with the ``scheme`` and ``host`` keys at a minimum.
  92. :type key_class: namedtuple
  93. :param request_context:
  94. A dictionary-like object that contain the context for a request.
  95. :type request_context: dict
  96. :return: A namedtuple that can be used as a connection pool key.
  97. :rtype: PoolKey
  98. """
  99. # Since we mutate the dictionary, make a copy first
  100. context = request_context.copy()
  101. context["scheme"] = context["scheme"].lower()
  102. context["host"] = context["host"].lower()
  103. # These are both dictionaries and need to be transformed into frozensets
  104. for key in ("headers", "_proxy_headers", "_socks_options"):
  105. if key in context and context[key] is not None:
  106. context[key] = frozenset(context[key].items())
  107. # The socket_options key may be a list and needs to be transformed into a
  108. # tuple.
  109. socket_opts = context.get("socket_options")
  110. if socket_opts is not None:
  111. context["socket_options"] = tuple(socket_opts)
  112. # Map the kwargs to the names in the namedtuple - this is necessary since
  113. # namedtuples can't have fields starting with '_'.
  114. for key in list(context.keys()):
  115. context["key_" + key] = context.pop(key)
  116. # Default to ``None`` for keys missing from the context
  117. for field in key_class._fields:
  118. if field not in context:
  119. context[field] = None
  120. # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
  121. if context.get("key_blocksize") is None:
  122. context["key_blocksize"] = _DEFAULT_BLOCKSIZE
  123. return key_class(**context)
  124. #: A dictionary that maps a scheme to a callable that creates a pool key.
  125. #: This can be used to alter the way pool keys are constructed, if desired.
  126. #: Each PoolManager makes a copy of this dictionary so they can be configured
  127. #: globally here, or individually on the instance.
  128. key_fn_by_scheme = {
  129. "http": functools.partial(_default_key_normalizer, PoolKey),
  130. "https": functools.partial(_default_key_normalizer, PoolKey),
  131. }
  132. pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
  133. class PoolManager(RequestMethods):
  134. """
  135. Allows for arbitrary requests while transparently keeping track of
  136. necessary connection pools for you.
  137. :param num_pools:
  138. Number of connection pools to cache before discarding the least
  139. recently used pool.
  140. :param headers:
  141. Headers to include with all requests, unless other headers are given
  142. explicitly.
  143. :param \\**connection_pool_kw:
  144. Additional parameters are used to create fresh
  145. :class:`urllib3.connectionpool.ConnectionPool` instances.
  146. Example:
  147. .. code-block:: python
  148. import urllib3
  149. http = urllib3.PoolManager(num_pools=2)
  150. resp1 = http.request("GET", "https://google.com/")
  151. resp2 = http.request("GET", "https://google.com/mail")
  152. resp3 = http.request("GET", "https://yahoo.com/")
  153. print(len(http.pools))
  154. # 2
  155. """
  156. proxy: Url | None = None
  157. proxy_config: ProxyConfig | None = None
  158. def __init__(
  159. self,
  160. num_pools: int = 10,
  161. headers: typing.Mapping[str, str] | None = None,
  162. **connection_pool_kw: typing.Any,
  163. ) -> None:
  164. super().__init__(headers)
  165. self.connection_pool_kw = connection_pool_kw
  166. self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
  167. self.pools = RecentlyUsedContainer(num_pools)
  168. # Locally set the pool classes and keys so other PoolManagers can
  169. # override them.
  170. self.pool_classes_by_scheme = pool_classes_by_scheme
  171. self.key_fn_by_scheme = key_fn_by_scheme.copy()
  172. def __enter__(self: _SelfT) -> _SelfT:
  173. return self
  174. def __exit__(
  175. self,
  176. exc_type: type[BaseException] | None,
  177. exc_val: BaseException | None,
  178. exc_tb: TracebackType | None,
  179. ) -> Literal[False]:
  180. self.clear()
  181. # Return False to re-raise any potential exceptions
  182. return False
  183. def _new_pool(
  184. self,
  185. scheme: str,
  186. host: str,
  187. port: int,
  188. request_context: dict[str, typing.Any] | None = None,
  189. ) -> HTTPConnectionPool:
  190. """
  191. Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
  192. any additional pool keyword arguments.
  193. If ``request_context`` is provided, it is provided as keyword arguments
  194. to the pool class used. This method is used to actually create the
  195. connection pools handed out by :meth:`connection_from_url` and
  196. companion methods. It is intended to be overridden for customization.
  197. """
  198. pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
  199. if request_context is None:
  200. request_context = self.connection_pool_kw.copy()
  201. # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
  202. # set to 'None' in the request_context.
  203. if request_context.get("blocksize") is None:
  204. request_context["blocksize"] = _DEFAULT_BLOCKSIZE
  205. # Although the context has everything necessary to create the pool,
  206. # this function has historically only used the scheme, host, and port
  207. # in the positional args. When an API change is acceptable these can
  208. # be removed.
  209. for key in ("scheme", "host", "port"):
  210. request_context.pop(key, None)
  211. if scheme == "http":
  212. for kw in SSL_KEYWORDS:
  213. request_context.pop(kw, None)
  214. return pool_cls(host, port, **request_context)
  215. def clear(self) -> None:
  216. """
  217. Empty our store of pools and direct them all to close.
  218. This will not affect in-flight connections, but they will not be
  219. re-used after completion.
  220. """
  221. self.pools.clear()
  222. def connection_from_host(
  223. self,
  224. host: str | None,
  225. port: int | None = None,
  226. scheme: str | None = "http",
  227. pool_kwargs: dict[str, typing.Any] | None = None,
  228. ) -> HTTPConnectionPool:
  229. """
  230. Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
  231. If ``port`` isn't given, it will be derived from the ``scheme`` using
  232. ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
  233. provided, it is merged with the instance's ``connection_pool_kw``
  234. variable and used to create the new connection pool, if one is
  235. needed.
  236. """
  237. if not host:
  238. raise LocationValueError("No host specified.")
  239. request_context = self._merge_pool_kwargs(pool_kwargs)
  240. request_context["scheme"] = scheme or "http"
  241. if not port:
  242. port = port_by_scheme.get(request_context["scheme"].lower(), 80)
  243. request_context["port"] = port
  244. request_context["host"] = host
  245. return self.connection_from_context(request_context)
  246. def connection_from_context(
  247. self, request_context: dict[str, typing.Any]
  248. ) -> HTTPConnectionPool:
  249. """
  250. Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
  251. ``request_context`` must at least contain the ``scheme`` key and its
  252. value must be a key in ``key_fn_by_scheme`` instance variable.
  253. """
  254. if "strict" in request_context:
  255. warnings.warn(
  256. "The 'strict' parameter is no longer needed on Python 3+. "
  257. "This will raise an error in urllib3 v2.1.0.",
  258. DeprecationWarning,
  259. )
  260. request_context.pop("strict")
  261. scheme = request_context["scheme"].lower()
  262. pool_key_constructor = self.key_fn_by_scheme.get(scheme)
  263. if not pool_key_constructor:
  264. raise URLSchemeUnknown(scheme)
  265. pool_key = pool_key_constructor(request_context)
  266. return self.connection_from_pool_key(pool_key, request_context=request_context)
  267. def connection_from_pool_key(
  268. self, pool_key: PoolKey, request_context: dict[str, typing.Any]
  269. ) -> HTTPConnectionPool:
  270. """
  271. Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
  272. ``pool_key`` should be a namedtuple that only contains immutable
  273. objects. At a minimum it must have the ``scheme``, ``host``, and
  274. ``port`` fields.
  275. """
  276. with self.pools.lock:
  277. # If the scheme, host, or port doesn't match existing open
  278. # connections, open a new ConnectionPool.
  279. pool = self.pools.get(pool_key)
  280. if pool:
  281. return pool
  282. # Make a fresh ConnectionPool of the desired type
  283. scheme = request_context["scheme"]
  284. host = request_context["host"]
  285. port = request_context["port"]
  286. pool = self._new_pool(scheme, host, port, request_context=request_context)
  287. self.pools[pool_key] = pool
  288. return pool
  289. def connection_from_url(
  290. self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
  291. ) -> HTTPConnectionPool:
  292. """
  293. Similar to :func:`urllib3.connectionpool.connection_from_url`.
  294. If ``pool_kwargs`` is not provided and a new pool needs to be
  295. constructed, ``self.connection_pool_kw`` is used to initialize
  296. the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
  297. is provided, it is used instead. Note that if a new pool does not
  298. need to be created for the request, the provided ``pool_kwargs`` are
  299. not used.
  300. """
  301. u = parse_url(url)
  302. return self.connection_from_host(
  303. u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
  304. )
  305. def _merge_pool_kwargs(
  306. self, override: dict[str, typing.Any] | None
  307. ) -> dict[str, typing.Any]:
  308. """
  309. Merge a dictionary of override values for self.connection_pool_kw.
  310. This does not modify self.connection_pool_kw and returns a new dict.
  311. Any keys in the override dictionary with a value of ``None`` are
  312. removed from the merged dictionary.
  313. """
  314. base_pool_kwargs = self.connection_pool_kw.copy()
  315. if override:
  316. for key, value in override.items():
  317. if value is None:
  318. try:
  319. del base_pool_kwargs[key]
  320. except KeyError:
  321. pass
  322. else:
  323. base_pool_kwargs[key] = value
  324. return base_pool_kwargs
  325. def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
  326. """
  327. Indicates if the proxy requires the complete destination URL in the
  328. request. Normally this is only needed when not using an HTTP CONNECT
  329. tunnel.
  330. """
  331. if self.proxy is None:
  332. return False
  333. return not connection_requires_http_tunnel(
  334. self.proxy, self.proxy_config, parsed_url.scheme
  335. )
  336. def urlopen( # type: ignore[override]
  337. self, method: str, url: str, redirect: bool = True, **kw: typing.Any
  338. ) -> BaseHTTPResponse:
  339. """
  340. Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
  341. with custom cross-host redirect logic and only sends the request-uri
  342. portion of the ``url``.
  343. The given ``url`` parameter must be absolute, such that an appropriate
  344. :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
  345. """
  346. u = parse_url(url)
  347. if u.scheme is None:
  348. warnings.warn(
  349. "URLs without a scheme (ie 'https://') are deprecated and will raise an error "
  350. "in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs "
  351. "start with 'https://' or 'http://'. Read more in this issue: "
  352. "https://github.com/urllib3/urllib3/issues/2920",
  353. category=DeprecationWarning,
  354. stacklevel=2,
  355. )
  356. conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
  357. kw["assert_same_host"] = False
  358. kw["redirect"] = False
  359. if "headers" not in kw:
  360. kw["headers"] = self.headers
  361. if self._proxy_requires_url_absolute_form(u):
  362. response = conn.urlopen(method, url, **kw)
  363. else:
  364. response = conn.urlopen(method, u.request_uri, **kw)
  365. redirect_location = redirect and response.get_redirect_location()
  366. if not redirect_location:
  367. return response
  368. # Support relative URLs for redirecting.
  369. redirect_location = urljoin(url, redirect_location)
  370. # RFC 7231, Section 6.4.4
  371. if response.status == 303:
  372. method = "GET"
  373. retries = kw.get("retries")
  374. if not isinstance(retries, Retry):
  375. retries = Retry.from_int(retries, redirect=redirect)
  376. # Strip headers marked as unsafe to forward to the redirected location.
  377. # Check remove_headers_on_redirect to avoid a potential network call within
  378. # conn.is_same_host() which may use socket.gethostbyname() in the future.
  379. if retries.remove_headers_on_redirect and not conn.is_same_host(
  380. redirect_location
  381. ):
  382. new_headers = kw["headers"].copy()
  383. for header in kw["headers"]:
  384. if header.lower() in retries.remove_headers_on_redirect:
  385. new_headers.pop(header, None)
  386. kw["headers"] = new_headers
  387. try:
  388. retries = retries.increment(method, url, response=response, _pool=conn)
  389. except MaxRetryError:
  390. if retries.raise_on_redirect:
  391. response.drain_conn()
  392. raise
  393. return response
  394. kw["retries"] = retries
  395. kw["redirect"] = redirect
  396. log.info("Redirecting %s -> %s", url, redirect_location)
  397. response.drain_conn()
  398. return self.urlopen(method, redirect_location, **kw)
  399. class ProxyManager(PoolManager):
  400. """
  401. Behaves just like :class:`PoolManager`, but sends all requests through
  402. the defined proxy, using the CONNECT method for HTTPS URLs.
  403. :param proxy_url:
  404. The URL of the proxy to be used.
  405. :param proxy_headers:
  406. A dictionary containing headers that will be sent to the proxy. In case
  407. of HTTP they are being sent with each request, while in the
  408. HTTPS/CONNECT case they are sent only once. Could be used for proxy
  409. authentication.
  410. :param proxy_ssl_context:
  411. The proxy SSL context is used to establish the TLS connection to the
  412. proxy when using HTTPS proxies.
  413. :param use_forwarding_for_https:
  414. (Defaults to False) If set to True will forward requests to the HTTPS
  415. proxy to be made on behalf of the client instead of creating a TLS
  416. tunnel via the CONNECT method. **Enabling this flag means that request
  417. and response headers and content will be visible from the HTTPS proxy**
  418. whereas tunneling keeps request and response headers and content
  419. private. IP address, target hostname, SNI, and port are always visible
  420. to an HTTPS proxy even when this flag is disabled.
  421. :param proxy_assert_hostname:
  422. The hostname of the certificate to verify against.
  423. :param proxy_assert_fingerprint:
  424. The fingerprint of the certificate to verify against.
  425. Example:
  426. .. code-block:: python
  427. import urllib3
  428. proxy = urllib3.ProxyManager("https://localhost:3128/")
  429. resp1 = proxy.request("GET", "https://google.com/")
  430. resp2 = proxy.request("GET", "https://httpbin.org/")
  431. print(len(proxy.pools))
  432. # 1
  433. resp3 = proxy.request("GET", "https://httpbin.org/")
  434. resp4 = proxy.request("GET", "https://twitter.com/")
  435. print(len(proxy.pools))
  436. # 3
  437. """
  438. def __init__(
  439. self,
  440. proxy_url: str,
  441. num_pools: int = 10,
  442. headers: typing.Mapping[str, str] | None = None,
  443. proxy_headers: typing.Mapping[str, str] | None = None,
  444. proxy_ssl_context: ssl.SSLContext | None = None,
  445. use_forwarding_for_https: bool = False,
  446. proxy_assert_hostname: None | str | Literal[False] = None,
  447. proxy_assert_fingerprint: str | None = None,
  448. **connection_pool_kw: typing.Any,
  449. ) -> None:
  450. if isinstance(proxy_url, HTTPConnectionPool):
  451. str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
  452. else:
  453. str_proxy_url = proxy_url
  454. proxy = parse_url(str_proxy_url)
  455. if proxy.scheme not in ("http", "https"):
  456. raise ProxySchemeUnknown(proxy.scheme)
  457. if not proxy.port:
  458. port = port_by_scheme.get(proxy.scheme, 80)
  459. proxy = proxy._replace(port=port)
  460. self.proxy = proxy
  461. self.proxy_headers = proxy_headers or {}
  462. self.proxy_ssl_context = proxy_ssl_context
  463. self.proxy_config = ProxyConfig(
  464. proxy_ssl_context,
  465. use_forwarding_for_https,
  466. proxy_assert_hostname,
  467. proxy_assert_fingerprint,
  468. )
  469. connection_pool_kw["_proxy"] = self.proxy
  470. connection_pool_kw["_proxy_headers"] = self.proxy_headers
  471. connection_pool_kw["_proxy_config"] = self.proxy_config
  472. super().__init__(num_pools, headers, **connection_pool_kw)
  473. def connection_from_host(
  474. self,
  475. host: str | None,
  476. port: int | None = None,
  477. scheme: str | None = "http",
  478. pool_kwargs: dict[str, typing.Any] | None = None,
  479. ) -> HTTPConnectionPool:
  480. if scheme == "https":
  481. return super().connection_from_host(
  482. host, port, scheme, pool_kwargs=pool_kwargs
  483. )
  484. return super().connection_from_host(
  485. self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
  486. )
  487. def _set_proxy_headers(
  488. self, url: str, headers: typing.Mapping[str, str] | None = None
  489. ) -> typing.Mapping[str, str]:
  490. """
  491. Sets headers needed by proxies: specifically, the Accept and Host
  492. headers. Only sets headers not provided by the user.
  493. """
  494. headers_ = {"Accept": "*/*"}
  495. netloc = parse_url(url).netloc
  496. if netloc:
  497. headers_["Host"] = netloc
  498. if headers:
  499. headers_.update(headers)
  500. return headers_
  501. def urlopen( # type: ignore[override]
  502. self, method: str, url: str, redirect: bool = True, **kw: typing.Any
  503. ) -> BaseHTTPResponse:
  504. "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
  505. u = parse_url(url)
  506. if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
  507. # For connections using HTTP CONNECT, httplib sets the necessary
  508. # headers on the CONNECT to the proxy. If we're not using CONNECT,
  509. # we'll definitely need to set 'Host' at the very least.
  510. headers = kw.get("headers", self.headers)
  511. kw["headers"] = self._set_proxy_headers(url, headers)
  512. return super().urlopen(method, url, redirect=redirect, **kw)
  513. def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
  514. return ProxyManager(proxy_url=url, **kw)