bro
This commit is contained in:
1 parent
de41a02749
commit
9af09cebd3
764 files changed
+54218
-78859
No files matched your search
@@ -12,21 +12,27 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import urllib.parse
|
||||
from collections.abc import Iterable, MutableMapping, Sequence
|
||||
from collections.abc import Callable, Iterable, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from html.parser import HTMLParser
|
||||
from optparse import Values
|
||||
from typing import (
|
||||
Callable,
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
)
|
||||
|
||||
from pip._vendor import requests
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
from pip._vendor.requests import Response
|
||||
from pip._vendor.requests.exceptions import RetryError, SSLError
|
||||
from pip._vendor.requests.exceptions import RetryError
|
||||
|
||||
from pip._internal.exceptions import NetworkConnectionError
|
||||
from pip._internal.exceptions import (
|
||||
ConnectionFailedError,
|
||||
ConnectionTimeoutError,
|
||||
NetworkConnectionError,
|
||||
ProxyConnectionError,
|
||||
SSLMissingError,
|
||||
SSLVerificationError,
|
||||
)
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.models.search_scope import SearchScope
|
||||
from pip._internal.network.session import PipSession
|
||||
@@ -80,7 +86,7 @@ def _ensure_api_header(response: Response) -> None:
|
||||
):
|
||||
return
|
||||
|
||||
raise _NotAPIContent(content_type, response.request.method)
|
||||
raise _NotAPIContent(content_type, response.request.method) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class _NotHTTP(Exception):
|
||||
@@ -105,7 +111,11 @@ def _ensure_api_response(url: str, session: PipSession) -> None:
|
||||
_ensure_api_header(resp)
|
||||
|
||||
|
||||
def _get_simple_response(url: str, session: PipSession) -> Response:
|
||||
def _get_simple_response(
|
||||
url: str,
|
||||
session: PipSession,
|
||||
force_revalidate: bool = False,
|
||||
) -> Response:
|
||||
"""Access an Simple API response with GET, and return the response.
|
||||
|
||||
This consists of three parts:
|
||||
@@ -123,32 +133,24 @@ def _get_simple_response(url: str, session: PipSession) -> Response:
|
||||
|
||||
logger.debug("Getting page %s", redact_auth_from_url(url))
|
||||
|
||||
resp = session.get(
|
||||
url,
|
||||
headers={
|
||||
"Accept": ", ".join(
|
||||
[
|
||||
"application/vnd.pypi.simple.v1+json",
|
||||
"application/vnd.pypi.simple.v1+html; q=0.1",
|
||||
"text/html; q=0.01",
|
||||
]
|
||||
),
|
||||
# We don't want to blindly returned cached data for
|
||||
# /simple/, because authors generally expecting that
|
||||
# twine upload && pip install will function, but if
|
||||
# they've done a pip install in the last ~10 minutes
|
||||
# it won't. Thus by setting this to zero we will not
|
||||
# blindly use any cached data, however the benefit of
|
||||
# using max-age=0 instead of no-cache, is that we will
|
||||
# still support conditional requests, so we will still
|
||||
# minimize traffic sent in cases where the page hasn't
|
||||
# changed at all, we will just always incur the round
|
||||
# trip for the conditional GET now instead of only
|
||||
# once per 10 minutes.
|
||||
# For more information, please see pypa/pip#5670.
|
||||
"Cache-Control": "max-age=0",
|
||||
},
|
||||
)
|
||||
headers = {
|
||||
"Accept": ", ".join(
|
||||
[
|
||||
"application/vnd.pypi.simple.v1+json",
|
||||
"application/vnd.pypi.simple.v1+html; q=0.1",
|
||||
"text/html; q=0.01",
|
||||
]
|
||||
),
|
||||
}
|
||||
|
||||
if force_revalidate:
|
||||
# Using max-age=0 rather than no-cache still supports conditional
|
||||
# requests, minimizing traffic when the page hasn't changed.
|
||||
# See pypa/pip#5670.
|
||||
logger.debug("Refreshing package index response.")
|
||||
headers["Cache-Control"] = "max-age=0"
|
||||
|
||||
resp = session.get(url, headers=headers)
|
||||
raise_for_status(resp)
|
||||
|
||||
# The check for archives above only works if the url ends with
|
||||
@@ -316,7 +318,12 @@ def _make_index_content(
|
||||
)
|
||||
|
||||
|
||||
def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | None:
|
||||
def _get_index_content(
|
||||
link: Link,
|
||||
*,
|
||||
session: PipSession,
|
||||
force_revalidate: bool = False,
|
||||
) -> IndexContent | None:
|
||||
url = link.url.split("#", 1)[0]
|
||||
|
||||
# Check for VCS schemes that do not support lookup as web pages.
|
||||
@@ -343,7 +350,9 @@ def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | Non
|
||||
logger.debug(" file: URL is directory, getting %s", url)
|
||||
|
||||
try:
|
||||
resp = _get_simple_response(url, session=session)
|
||||
resp = _get_simple_response(
|
||||
url, session=session, force_revalidate=force_revalidate
|
||||
)
|
||||
except _NotHTTP:
|
||||
logger.warning(
|
||||
"Skipping page %s because it looks like an archive, and cannot "
|
||||
@@ -359,18 +368,17 @@ def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | Non
|
||||
exc.request_desc,
|
||||
exc.content_type,
|
||||
)
|
||||
except NetworkConnectionError as exc:
|
||||
except (RetryError, NetworkConnectionError) as exc:
|
||||
_handle_get_simple_fail(link, exc)
|
||||
except RetryError as exc:
|
||||
_handle_get_simple_fail(link, exc)
|
||||
except SSLError as exc:
|
||||
reason = "There was a problem confirming the ssl certificate: "
|
||||
reason += str(exc)
|
||||
except (SSLVerificationError, SSLMissingError) as exc:
|
||||
reason = f"There was a problem confirming the ssl certificate: {exc.context}"
|
||||
_handle_get_simple_fail(link, reason, meth=logger.info)
|
||||
except requests.ConnectionError as exc:
|
||||
_handle_get_simple_fail(link, f"connection error: {exc}")
|
||||
except requests.Timeout:
|
||||
_handle_get_simple_fail(link, "timed out")
|
||||
except ConnectionFailedError as exc:
|
||||
_handle_get_simple_fail(link, f"connection error: {exc.context}")
|
||||
except ProxyConnectionError as exc:
|
||||
_handle_get_simple_fail(link, f"proxy connection error: {exc.context}")
|
||||
except ConnectionTimeoutError as exc:
|
||||
_handle_get_simple_fail(link, str(exc.context))
|
||||
else:
|
||||
return _make_index_content(resp, cache_link_parsing=link.cache_link_parsing)
|
||||
return None
|
||||
@@ -435,11 +443,25 @@ class LinkCollector:
|
||||
def find_links(self) -> list[str]:
|
||||
return self.search_scope.find_links
|
||||
|
||||
def fetch_response(self, location: Link) -> IndexContent | None:
|
||||
def fetch_response(
|
||||
self, location: Link, package_name: str | None = None
|
||||
) -> IndexContent | None:
|
||||
"""
|
||||
Fetch an HTML page containing package links.
|
||||
"""
|
||||
return _get_index_content(location, session=self.session)
|
||||
# By default we may return cached /simple/ responses. Users can force
|
||||
# revalidation to ensure newly uploaded distributions become visible
|
||||
# immediately after publication. See pypa/pip#5670.
|
||||
force_revalidate = self.session.refresh_package
|
||||
should_force_revalidate = ":all:" in force_revalidate or (
|
||||
package_name is not None
|
||||
and canonicalize_name(package_name) in force_revalidate
|
||||
)
|
||||
return _get_index_content(
|
||||
location,
|
||||
session=self.session,
|
||||
force_revalidate=should_force_revalidate,
|
||||
)
|
||||
|
||||
def collect_sources(
|
||||
self,
|
||||
|
||||
@@ -12,14 +12,12 @@ from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
|
||||
from pip._vendor.packaging import specifiers
|
||||
from pip._vendor.packaging.tags import Tag
|
||||
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
|
||||
from pip._vendor.packaging.version import InvalidVersion, Version, _BaseVersion
|
||||
from pip._vendor.packaging.version import InvalidVersion, _BaseVersion
|
||||
from pip._vendor.packaging.version import parse as parse_version
|
||||
|
||||
from pip._internal.exceptions import (
|
||||
@@ -49,15 +47,15 @@ from pip._internal.utils.packaging import check_requires_python
|
||||
from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing_extensions import TypeGuard
|
||||
from typing import TypeGuard
|
||||
|
||||
__all__ = ["FormatControl", "BestCandidateResult", "PackageFinder"]
|
||||
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
BuildTag = Union[tuple[()], tuple[int, str]]
|
||||
CandidateSortingKey = tuple[int, int, int, _BaseVersion, Optional[int], BuildTag]
|
||||
BuildTag = tuple[()] | tuple[int, str]
|
||||
CandidateSortingKey = tuple[int, int, int, _BaseVersion, int | None, BuildTag]
|
||||
|
||||
|
||||
def _check_link_requires_python(
|
||||
@@ -137,7 +135,7 @@ class LinkEvaluator:
|
||||
formats: frozenset[str],
|
||||
target_python: TargetPython,
|
||||
allow_yanked: bool,
|
||||
ignore_requires_python: bool | None = None,
|
||||
ignore_requires_python: bool = False,
|
||||
uploaded_prior_to: datetime.datetime | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -159,8 +157,6 @@ class LinkEvaluator:
|
||||
:param uploaded_prior_to: If set, only allow links uploaded prior to
|
||||
the given datetime.
|
||||
"""
|
||||
if ignore_requires_python is None:
|
||||
ignore_requires_python = False
|
||||
|
||||
self._allow_yanked = allow_yanked
|
||||
self._canonical_name = canonical_name
|
||||
@@ -493,7 +489,6 @@ class CandidateEvaluator:
|
||||
else:
|
||||
allow_prereleases = None
|
||||
specifier = self._specifier
|
||||
|
||||
# When using the pkg_resources backend we turn the version object into
|
||||
# a str here because otherwise when we're debundled but setuptools isn't,
|
||||
# Python will see packaging.version.Version and
|
||||
@@ -501,22 +496,17 @@ class CandidateEvaluator:
|
||||
# types. This way we'll use a str as a common data interchange
|
||||
# format. If we stop using the pkg_resources provided specifier
|
||||
# and start using our own, we can drop the cast to str().
|
||||
if select_backend().NAME == "pkg_resources":
|
||||
candidates_and_versions: list[
|
||||
tuple[InstallationCandidate, str | Version]
|
||||
] = [(c, str(c.version)) for c in candidates]
|
||||
else:
|
||||
candidates_and_versions = [(c, c.version) for c in candidates]
|
||||
versions = set(
|
||||
specifier.filter(
|
||||
(v for _, v in candidates_and_versions),
|
||||
prereleases=allow_prereleases,
|
||||
)
|
||||
applicable_candidates = specifier.filter(
|
||||
candidates,
|
||||
prereleases=allow_prereleases,
|
||||
key=lambda c: (
|
||||
str(c.version)
|
||||
if select_backend().NAME == "pkg_resources"
|
||||
else c.version
|
||||
),
|
||||
)
|
||||
|
||||
applicable_candidates = [c for c, v in candidates_and_versions if v in versions]
|
||||
filtered_applicable_candidates = filter_unallowed_hashes(
|
||||
candidates=applicable_candidates,
|
||||
candidates=list(applicable_candidates),
|
||||
hashes=self._hashes,
|
||||
project_name=self._project_name,
|
||||
)
|
||||
@@ -633,7 +623,7 @@ class PackageFinder:
|
||||
allow_yanked: bool,
|
||||
format_control: FormatControl | None = None,
|
||||
candidate_prefs: CandidatePreferences | None = None,
|
||||
ignore_requires_python: bool | None = None,
|
||||
ignore_requires_python: bool = False,
|
||||
uploaded_prior_to: datetime.datetime | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -660,8 +650,10 @@ class PackageFinder:
|
||||
|
||||
self.format_control = format_control
|
||||
|
||||
# These are boring links that have already been logged somehow.
|
||||
self._logged_links: set[tuple[Link, LinkType, str]] = set()
|
||||
# Collects the detail strings for links skipped due to Requires-Python
|
||||
# incompatibility. Used by requires_python_skipped_reasons() to build
|
||||
# the error message when resolution fails.
|
||||
self._requires_python_skipped: set[str] = set()
|
||||
|
||||
# Cache of the result of finding candidates
|
||||
self._all_candidates: dict[str, list[InstallationCandidate]] = {}
|
||||
@@ -670,6 +662,9 @@ class PackageFinder:
|
||||
BestCandidateResult,
|
||||
] = {}
|
||||
|
||||
# projects for which a link is locked from a pylock
|
||||
self._locked_links: dict[NormalizedName, Link] = {}
|
||||
|
||||
# Don't include an allow_yanked default value to make sure each call
|
||||
# site considers whether yanked releases are allowed. This also causes
|
||||
# that decision to be made explicit in the calling code, which helps
|
||||
@@ -730,10 +725,18 @@ class PackageFinder:
|
||||
def index_urls(self) -> list[str]:
|
||||
return self.search_scope.index_urls
|
||||
|
||||
@property
|
||||
def refresh_package(self) -> set[str]:
|
||||
return self._link_collector.session.refresh_package
|
||||
|
||||
@property
|
||||
def proxy(self) -> str | None:
|
||||
return self._link_collector.session.pip_proxy
|
||||
|
||||
@property
|
||||
def no_proxy_env(self) -> bool:
|
||||
return self._link_collector.session.pip_no_proxy_env
|
||||
|
||||
@property
|
||||
def trusted_hosts(self) -> Iterable[str]:
|
||||
for host_port in self._link_collector.session.pip_trusted_origins:
|
||||
@@ -772,12 +775,7 @@ class PackageFinder:
|
||||
return self._uploaded_prior_to
|
||||
|
||||
def requires_python_skipped_reasons(self) -> list[str]:
|
||||
reasons = {
|
||||
detail
|
||||
for _, result, detail in self._logged_links
|
||||
if result == LinkType.requires_python_mismatch
|
||||
}
|
||||
return sorted(reasons)
|
||||
return sorted(self._requires_python_skipped)
|
||||
|
||||
def make_link_evaluator(self, project_name: str) -> LinkEvaluator:
|
||||
canonical_name = canonicalize_name(project_name)
|
||||
@@ -810,12 +808,11 @@ class PackageFinder:
|
||||
return no_eggs + eggs
|
||||
|
||||
def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None:
|
||||
entry = (link, result, detail)
|
||||
if entry not in self._logged_links:
|
||||
# Put the link at the end so the reason is more visible and because
|
||||
# the link string is usually very long.
|
||||
logger.debug("Skipping link: %s: %s", detail, link)
|
||||
self._logged_links.add(entry)
|
||||
# Put the link at the end so the reason is more visible and because
|
||||
# the link string is usually very long.
|
||||
logger.debug("Skipping link: %s: %s", detail, link)
|
||||
if result == LinkType.requires_python_mismatch:
|
||||
self._requires_python_skipped.add(detail)
|
||||
|
||||
def get_install_candidate(
|
||||
self, link_evaluator: LinkEvaluator, link: Link
|
||||
@@ -863,7 +860,9 @@ class PackageFinder:
|
||||
"Fetching project page and analyzing links: %s",
|
||||
project_url,
|
||||
)
|
||||
index_response = self._link_collector.fetch_response(project_url)
|
||||
index_response = self._link_collector.fetch_response(
|
||||
project_url, package_name=link_evaluator.project_name
|
||||
)
|
||||
if index_response is None:
|
||||
return []
|
||||
|
||||
@@ -880,7 +879,8 @@ class PackageFinder:
|
||||
def find_all_candidates(self, project_name: str) -> list[InstallationCandidate]:
|
||||
"""Find all available InstallationCandidate for project_name
|
||||
|
||||
This checks index_urls and find_links.
|
||||
This checks index_urls and find_links, unless a locked link is known
|
||||
for that project.
|
||||
All versions found are returned as an InstallationCandidate list.
|
||||
|
||||
See LinkEvaluator.evaluate_link() for details on which files
|
||||
@@ -891,6 +891,23 @@ class PackageFinder:
|
||||
|
||||
link_evaluator = self.make_link_evaluator(project_name)
|
||||
|
||||
if locked_link := self._locked_links.get(canonicalize_name(project_name)):
|
||||
# If a locked link is known for that project, do not check
|
||||
# index_urls nor find_links. We don't use get_install_candidate here,
|
||||
# because if a locked link is unsupported (due to format control,
|
||||
# release control or otherwise), we want to error out immediately
|
||||
# instead of ignoring it.
|
||||
result, detail = link_evaluator.evaluate_link(locked_link)
|
||||
if result != LinkType.candidate:
|
||||
raise InstallationError(
|
||||
f"Could not install locked package {project_name!r} "
|
||||
f"from {locked_link.comes_from!r}: {detail}"
|
||||
)
|
||||
self._all_candidates[project_name] = [
|
||||
InstallationCandidate(project_name, detail, locked_link, locked=True)
|
||||
]
|
||||
return self._all_candidates[project_name]
|
||||
|
||||
collected_sources = self._link_collector.collect_sources(
|
||||
project_name=project_name,
|
||||
candidates_from_page=functools.partial(
|
||||
@@ -1028,16 +1045,16 @@ class PackageFinder:
|
||||
)
|
||||
if allows_pre is False:
|
||||
version_type = "final version"
|
||||
|
||||
req_disp = req.format_without_marker()
|
||||
logger.critical(
|
||||
"Could not find a %s that satisfies the requirement %s "
|
||||
"(from versions: %s)",
|
||||
version_type,
|
||||
req,
|
||||
req_disp,
|
||||
_format_versions(best_candidate_result.all_candidates),
|
||||
)
|
||||
|
||||
raise DistributionNotFound(f"No matching distribution found for {req}")
|
||||
raise DistributionNotFound(f"No matching distribution found for {req_disp}")
|
||||
|
||||
def _should_install_candidate(
|
||||
candidate: InstallationCandidate | None,
|
||||
@@ -1080,6 +1097,16 @@ class PackageFinder:
|
||||
)
|
||||
raise BestVersionAlreadyInstalled
|
||||
|
||||
def add_locked_link(self, project_name: NormalizedName, locked_link: Link) -> None:
|
||||
assert not self._all_candidates
|
||||
if project_name in self._locked_links:
|
||||
raise InstallationError(
|
||||
f"Multiple locked links provided for {project_name}: "
|
||||
f"{self._locked_links[project_name]} and {locked_link}"
|
||||
)
|
||||
|
||||
self._locked_links[project_name] = locked_link
|
||||
|
||||
|
||||
def _find_name_version_sep(fragment: str, canonical_name: str) -> int:
|
||||
"""Find the separator's index based on the package's canonical name.
|
||||
|
||||
@@ -4,8 +4,7 @@ import logging
|
||||
import mimetypes
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from typing import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from pip._vendor.packaging.utils import (
|
||||
InvalidSdistFilename,
|
||||
|
||||
Reference in new issue
Block a user