This commit is contained in:
wittmannzl103469 committed 2026-11-05 15:08:42 +01:00
1 parent de41a02749
commit 9af09cebd3
764 files changed
+54218 -78859

No files matched your search

@@ -12,21 +12,27 @@ import json
import logging
import os
import urllib.parse
from collections.abc import Iterable, MutableMapping, Sequence
from collections.abc import Callable, Iterable, MutableMapping, Sequence
from dataclasses import dataclass
from html.parser import HTMLParser
from optparse import Values
from typing import (
Callable,
NamedTuple,
Protocol,
)
from pip._vendor import requests
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.requests import Response
from pip._vendor.requests.exceptions import RetryError, SSLError
from pip._vendor.requests.exceptions import RetryError
from pip._internal.exceptions import NetworkConnectionError
from pip._internal.exceptions import (
ConnectionFailedError,
ConnectionTimeoutError,
NetworkConnectionError,
ProxyConnectionError,
SSLMissingError,
SSLVerificationError,
)
from pip._internal.models.link import Link
from pip._internal.models.search_scope import SearchScope
from pip._internal.network.session import PipSession
@@ -80,7 +86,7 @@ def _ensure_api_header(response: Response) -> None:
):
return
raise _NotAPIContent(content_type, response.request.method)
raise _NotAPIContent(content_type, response.request.method) # type: ignore[arg-type]
class _NotHTTP(Exception):
@@ -105,7 +111,11 @@ def _ensure_api_response(url: str, session: PipSession) -> None:
_ensure_api_header(resp)
def _get_simple_response(url: str, session: PipSession) -> Response:
def _get_simple_response(
url: str,
session: PipSession,
force_revalidate: bool = False,
) -> Response:
"""Access an Simple API response with GET, and return the response.
This consists of three parts:
@@ -123,32 +133,24 @@ def _get_simple_response(url: str, session: PipSession) -> Response:
logger.debug("Getting page %s", redact_auth_from_url(url))
resp = session.get(
url,
headers={
"Accept": ", ".join(
[
"application/vnd.pypi.simple.v1+json",
"application/vnd.pypi.simple.v1+html; q=0.1",
"text/html; q=0.01",
]
),
# We don't want to blindly returned cached data for
# /simple/, because authors generally expecting that
# twine upload && pip install will function, but if
# they've done a pip install in the last ~10 minutes
# it won't. Thus by setting this to zero we will not
# blindly use any cached data, however the benefit of
# using max-age=0 instead of no-cache, is that we will
# still support conditional requests, so we will still
# minimize traffic sent in cases where the page hasn't
# changed at all, we will just always incur the round
# trip for the conditional GET now instead of only
# once per 10 minutes.
# For more information, please see pypa/pip#5670.
"Cache-Control": "max-age=0",
},
)
headers = {
"Accept": ", ".join(
[
"application/vnd.pypi.simple.v1+json",
"application/vnd.pypi.simple.v1+html; q=0.1",
"text/html; q=0.01",
]
),
}
if force_revalidate:
# Using max-age=0 rather than no-cache still supports conditional
# requests, minimizing traffic when the page hasn't changed.
# See pypa/pip#5670.
logger.debug("Refreshing package index response.")
headers["Cache-Control"] = "max-age=0"
resp = session.get(url, headers=headers)
raise_for_status(resp)
# The check for archives above only works if the url ends with
@@ -316,7 +318,12 @@ def _make_index_content(
)
def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | None:
def _get_index_content(
link: Link,
*,
session: PipSession,
force_revalidate: bool = False,
) -> IndexContent | None:
url = link.url.split("#", 1)[0]
# Check for VCS schemes that do not support lookup as web pages.
@@ -343,7 +350,9 @@ def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | Non
logger.debug(" file: URL is directory, getting %s", url)
try:
resp = _get_simple_response(url, session=session)
resp = _get_simple_response(
url, session=session, force_revalidate=force_revalidate
)
except _NotHTTP:
logger.warning(
"Skipping page %s because it looks like an archive, and cannot "
@@ -359,18 +368,17 @@ def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | Non
exc.request_desc,
exc.content_type,
)
except NetworkConnectionError as exc:
except (RetryError, NetworkConnectionError) as exc:
_handle_get_simple_fail(link, exc)
except RetryError as exc:
_handle_get_simple_fail(link, exc)
except SSLError as exc:
reason = "There was a problem confirming the ssl certificate: "
reason += str(exc)
except (SSLVerificationError, SSLMissingError) as exc:
reason = f"There was a problem confirming the ssl certificate: {exc.context}"
_handle_get_simple_fail(link, reason, meth=logger.info)
except requests.ConnectionError as exc:
_handle_get_simple_fail(link, f"connection error: {exc}")
except requests.Timeout:
_handle_get_simple_fail(link, "timed out")
except ConnectionFailedError as exc:
_handle_get_simple_fail(link, f"connection error: {exc.context}")
except ProxyConnectionError as exc:
_handle_get_simple_fail(link, f"proxy connection error: {exc.context}")
except ConnectionTimeoutError as exc:
_handle_get_simple_fail(link, str(exc.context))
else:
return _make_index_content(resp, cache_link_parsing=link.cache_link_parsing)
return None
@@ -435,11 +443,25 @@ class LinkCollector:
def find_links(self) -> list[str]:
return self.search_scope.find_links
def fetch_response(self, location: Link) -> IndexContent | None:
def fetch_response(
self, location: Link, package_name: str | None = None
) -> IndexContent | None:
"""
Fetch an HTML page containing package links.
"""
return _get_index_content(location, session=self.session)
# By default we may return cached /simple/ responses. Users can force
# revalidation to ensure newly uploaded distributions become visible
# immediately after publication. See pypa/pip#5670.
force_revalidate = self.session.refresh_package
should_force_revalidate = ":all:" in force_revalidate or (
package_name is not None
and canonicalize_name(package_name) in force_revalidate
)
return _get_index_content(
location,
session=self.session,
force_revalidate=should_force_revalidate,
)
def collect_sources(
self,
@@ -12,14 +12,12 @@ from collections.abc import Iterable
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Optional,
Union,
)
from pip._vendor.packaging import specifiers
from pip._vendor.packaging.tags import Tag
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import InvalidVersion, Version, _BaseVersion
from pip._vendor.packaging.version import InvalidVersion, _BaseVersion
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import (
@@ -49,15 +47,15 @@ from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
if TYPE_CHECKING:
from typing_extensions import TypeGuard
from typing import TypeGuard
__all__ = ["FormatControl", "BestCandidateResult", "PackageFinder"]
logger = getLogger(__name__)
BuildTag = Union[tuple[()], tuple[int, str]]
CandidateSortingKey = tuple[int, int, int, _BaseVersion, Optional[int], BuildTag]
BuildTag = tuple[()] | tuple[int, str]
CandidateSortingKey = tuple[int, int, int, _BaseVersion, int | None, BuildTag]
def _check_link_requires_python(
@@ -137,7 +135,7 @@ class LinkEvaluator:
formats: frozenset[str],
target_python: TargetPython,
allow_yanked: bool,
ignore_requires_python: bool | None = None,
ignore_requires_python: bool = False,
uploaded_prior_to: datetime.datetime | None = None,
) -> None:
"""
@@ -159,8 +157,6 @@ class LinkEvaluator:
:param uploaded_prior_to: If set, only allow links uploaded prior to
the given datetime.
"""
if ignore_requires_python is None:
ignore_requires_python = False
self._allow_yanked = allow_yanked
self._canonical_name = canonical_name
@@ -493,7 +489,6 @@ class CandidateEvaluator:
else:
allow_prereleases = None
specifier = self._specifier
# When using the pkg_resources backend we turn the version object into
# a str here because otherwise when we're debundled but setuptools isn't,
# Python will see packaging.version.Version and
@@ -501,22 +496,17 @@ class CandidateEvaluator:
# types. This way we'll use a str as a common data interchange
# format. If we stop using the pkg_resources provided specifier
# and start using our own, we can drop the cast to str().
if select_backend().NAME == "pkg_resources":
candidates_and_versions: list[
tuple[InstallationCandidate, str | Version]
] = [(c, str(c.version)) for c in candidates]
else:
candidates_and_versions = [(c, c.version) for c in candidates]
versions = set(
specifier.filter(
(v for _, v in candidates_and_versions),
prereleases=allow_prereleases,
)
applicable_candidates = specifier.filter(
candidates,
prereleases=allow_prereleases,
key=lambda c: (
str(c.version)
if select_backend().NAME == "pkg_resources"
else c.version
),
)
applicable_candidates = [c for c, v in candidates_and_versions if v in versions]
filtered_applicable_candidates = filter_unallowed_hashes(
candidates=applicable_candidates,
candidates=list(applicable_candidates),
hashes=self._hashes,
project_name=self._project_name,
)
@@ -633,7 +623,7 @@ class PackageFinder:
allow_yanked: bool,
format_control: FormatControl | None = None,
candidate_prefs: CandidatePreferences | None = None,
ignore_requires_python: bool | None = None,
ignore_requires_python: bool = False,
uploaded_prior_to: datetime.datetime | None = None,
) -> None:
"""
@@ -660,8 +650,10 @@ class PackageFinder:
self.format_control = format_control
# These are boring links that have already been logged somehow.
self._logged_links: set[tuple[Link, LinkType, str]] = set()
# Collects the detail strings for links skipped due to Requires-Python
# incompatibility. Used by requires_python_skipped_reasons() to build
# the error message when resolution fails.
self._requires_python_skipped: set[str] = set()
# Cache of the result of finding candidates
self._all_candidates: dict[str, list[InstallationCandidate]] = {}
@@ -670,6 +662,9 @@ class PackageFinder:
BestCandidateResult,
] = {}
# projects for which a link is locked from a pylock
self._locked_links: dict[NormalizedName, Link] = {}
# Don't include an allow_yanked default value to make sure each call
# site considers whether yanked releases are allowed. This also causes
# that decision to be made explicit in the calling code, which helps
@@ -730,10 +725,18 @@ class PackageFinder:
def index_urls(self) -> list[str]:
return self.search_scope.index_urls
@property
def refresh_package(self) -> set[str]:
return self._link_collector.session.refresh_package
@property
def proxy(self) -> str | None:
return self._link_collector.session.pip_proxy
@property
def no_proxy_env(self) -> bool:
return self._link_collector.session.pip_no_proxy_env
@property
def trusted_hosts(self) -> Iterable[str]:
for host_port in self._link_collector.session.pip_trusted_origins:
@@ -772,12 +775,7 @@ class PackageFinder:
return self._uploaded_prior_to
def requires_python_skipped_reasons(self) -> list[str]:
reasons = {
detail
for _, result, detail in self._logged_links
if result == LinkType.requires_python_mismatch
}
return sorted(reasons)
return sorted(self._requires_python_skipped)
def make_link_evaluator(self, project_name: str) -> LinkEvaluator:
canonical_name = canonicalize_name(project_name)
@@ -810,12 +808,11 @@ class PackageFinder:
return no_eggs + eggs
def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None:
entry = (link, result, detail)
if entry not in self._logged_links:
# Put the link at the end so the reason is more visible and because
# the link string is usually very long.
logger.debug("Skipping link: %s: %s", detail, link)
self._logged_links.add(entry)
# Put the link at the end so the reason is more visible and because
# the link string is usually very long.
logger.debug("Skipping link: %s: %s", detail, link)
if result == LinkType.requires_python_mismatch:
self._requires_python_skipped.add(detail)
def get_install_candidate(
self, link_evaluator: LinkEvaluator, link: Link
@@ -863,7 +860,9 @@ class PackageFinder:
"Fetching project page and analyzing links: %s",
project_url,
)
index_response = self._link_collector.fetch_response(project_url)
index_response = self._link_collector.fetch_response(
project_url, package_name=link_evaluator.project_name
)
if index_response is None:
return []
@@ -880,7 +879,8 @@ class PackageFinder:
def find_all_candidates(self, project_name: str) -> list[InstallationCandidate]:
"""Find all available InstallationCandidate for project_name
This checks index_urls and find_links.
This checks index_urls and find_links, unless a locked link is known
for that project.
All versions found are returned as an InstallationCandidate list.
See LinkEvaluator.evaluate_link() for details on which files
@@ -891,6 +891,23 @@ class PackageFinder:
link_evaluator = self.make_link_evaluator(project_name)
if locked_link := self._locked_links.get(canonicalize_name(project_name)):
# If a locked link is known for that project, do not check
# index_urls nor find_links. We don't use get_install_candidate here,
# because if a locked link is unsupported (due to format control,
# release control or otherwise), we want to error out immediately
# instead of ignoring it.
result, detail = link_evaluator.evaluate_link(locked_link)
if result != LinkType.candidate:
raise InstallationError(
f"Could not install locked package {project_name!r} "
f"from {locked_link.comes_from!r}: {detail}"
)
self._all_candidates[project_name] = [
InstallationCandidate(project_name, detail, locked_link, locked=True)
]
return self._all_candidates[project_name]
collected_sources = self._link_collector.collect_sources(
project_name=project_name,
candidates_from_page=functools.partial(
@@ -1028,16 +1045,16 @@ class PackageFinder:
)
if allows_pre is False:
version_type = "final version"
req_disp = req.format_without_marker()
logger.critical(
"Could not find a %s that satisfies the requirement %s "
"(from versions: %s)",
version_type,
req,
req_disp,
_format_versions(best_candidate_result.all_candidates),
)
raise DistributionNotFound(f"No matching distribution found for {req}")
raise DistributionNotFound(f"No matching distribution found for {req_disp}")
def _should_install_candidate(
candidate: InstallationCandidate | None,
@@ -1080,6 +1097,16 @@ class PackageFinder:
)
raise BestVersionAlreadyInstalled
def add_locked_link(self, project_name: NormalizedName, locked_link: Link) -> None:
assert not self._all_candidates
if project_name in self._locked_links:
raise InstallationError(
f"Multiple locked links provided for {project_name}: "
f"{self._locked_links[project_name]} and {locked_link}"
)
self._locked_links[project_name] = locked_link
def _find_name_version_sep(fragment: str, canonical_name: str) -> int:
"""Find the separator's index based on the package's canonical name.
@@ -4,8 +4,7 @@ import logging
import mimetypes
import os
from collections import defaultdict
from collections.abc import Iterable
from typing import Callable
from collections.abc import Callable, Iterable
from pip._vendor.packaging.utils import (
InvalidSdistFilename,