This commit is contained in:
wittmannzl103469 committed 2026-11-05 15:08:42 +01:00
1 parent de41a02749
commit 9af09cebd3
764 files changed
+54218 -78859

No files matched your search

View File
File mode changed.
@@ -1,606 +0,0 @@
"""Build Environment used for isolation during sdist building"""
from __future__ import annotations
import logging
import os
import pathlib
import site
import sys
import textwrap
from collections import OrderedDict
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager as ContextManager
from contextlib import nullcontext
from io import StringIO
from types import TracebackType
from typing import TYPE_CHECKING, Protocol, TypedDict
from pip._vendor.packaging.version import Version
from pip import __file__ as pip_location
from pip._internal.cli.spinners import open_rich_spinner, open_spinner
from pip._internal.exceptions import (
BuildDependencyInstallError,
DiagnosticPipError,
InstallWheelBuildError,
PipError,
)
from pip._internal.locations import get_platlib, get_purelib, get_scheme
from pip._internal.metadata import get_default_environment, get_environment
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.logging import VERBOSE, capture_logging
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.subprocess import call_subprocess
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
if TYPE_CHECKING:
from pip._internal.cache import WheelCache
from pip._internal.index.package_finder import PackageFinder
from pip._internal.operations.build.build_tracker import BuildTracker
from pip._internal.req.req_install import InstallRequirement
from pip._internal.resolution.base import BaseResolver
class ExtraEnviron(TypedDict, total=False):
extra_environ: dict[str, str]
logger = logging.getLogger(__name__)
def _dedup(a: str, b: str) -> tuple[str] | tuple[str, str]:
return (a, b) if a != b else (a,)
class _Prefix:
def __init__(self, path: str) -> None:
self.path = path
self.setup = False
scheme = get_scheme("", prefix=path)
self.bin_dir = scheme.scripts
self.lib_dirs = _dedup(scheme.purelib, scheme.platlib)
def get_runnable_pip() -> str:
"""Get a file to pass to a Python executable, to run the currently-running pip.
This is used to run a pip subprocess, for installing requirements into the build
environment.
"""
source = pathlib.Path(pip_location).resolve().parent
if not source.is_dir():
# This would happen if someone is using pip from inside a zip file. In that
# case, we can use that directly.
return str(source)
return os.fsdecode(source / "__pip-runner__.py")
def _get_system_sitepackages() -> set[str]:
"""Get system site packages
Usually from site.getsitepackages,
but fallback on `get_purelib()/get_platlib()` if unavailable
(e.g. in a virtualenv created by virtualenv<20)
Returns normalized set of strings.
"""
if hasattr(site, "getsitepackages"):
system_sites = site.getsitepackages()
else:
# virtualenv < 20 overwrites site.py without getsitepackages
# fallback on get_purelib/get_platlib.
# this is known to miss things, but shouldn't in the cases
# where getsitepackages() has been removed (inside a virtualenv)
system_sites = [get_purelib(), get_platlib()]
return {os.path.normcase(path) for path in system_sites}
class BuildEnvironmentInstaller(Protocol):
"""
Interface for installing build dependencies into an isolated build
environment.
"""
def install(
self,
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
for_req: InstallRequirement | None,
) -> None: ...
class SubprocessBuildEnvironmentInstaller:
"""
Install build dependencies by calling pip in a subprocess.
"""
def __init__(
self,
finder: PackageFinder,
build_constraints: list[str] | None = None,
build_constraint_feature_enabled: bool = False,
) -> None:
self.finder = finder
self._build_constraints = build_constraints or []
self._build_constraint_feature_enabled = build_constraint_feature_enabled
def _deprecation_constraint_check(self) -> None:
"""
Check for deprecation warning: PIP_CONSTRAINT affecting build environments.
This warns when build-constraint feature is NOT enabled and PIP_CONSTRAINT
is not empty.
"""
if self._build_constraint_feature_enabled or self._build_constraints:
return
pip_constraint = os.environ.get("PIP_CONSTRAINT")
if not pip_constraint or not pip_constraint.strip():
return
deprecated(
reason=(
"Setting PIP_CONSTRAINT will not affect "
"build constraints in the future,"
),
replacement=(
"to specify build constraints using --build-constraint or "
"PIP_BUILD_CONSTRAINT. To disable this warning without "
"any build constraints set --use-feature=build-constraint or "
'PIP_USE_FEATURE="build-constraint"'
),
gone_in="26.2",
issue=None,
)
def install(
self,
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
for_req: InstallRequirement | None,
) -> None:
self._deprecation_constraint_check()
finder = self.finder
args: list[str] = [
sys.executable,
get_runnable_pip(),
"install",
"--ignore-installed",
"--no-user",
"--prefix",
prefix.path,
"--no-warn-script-location",
"--disable-pip-version-check",
# As the build environment is ephemeral, it's wasteful to
# pre-compile everything, especially as not every Python
# module will be used/compiled in most cases.
"--no-compile",
# The prefix specified two lines above, thus
# target from config file or env var should be ignored
"--target",
"",
]
if logger.getEffectiveLevel() <= logging.DEBUG:
args.append("-vv")
elif logger.getEffectiveLevel() <= VERBOSE:
args.append("-v")
for format_control in ("no_binary", "only_binary"):
formats = getattr(finder.format_control, format_control)
args.extend(
(
"--" + format_control.replace("_", "-"),
",".join(sorted(formats or {":none:"})),
)
)
if finder.release_control is not None:
# Use ordered args to preserve the user's original command-line order
# This is important because later flags can override earlier ones
for attr_name, value in finder.release_control.get_ordered_args():
args.extend(("--" + attr_name.replace("_", "-"), value))
index_urls = finder.index_urls
if index_urls:
args.extend(["-i", index_urls[0]])
for extra_index in index_urls[1:]:
args.extend(["--extra-index-url", extra_index])
else:
args.append("--no-index")
for link in finder.find_links:
args.extend(["--find-links", link])
if finder.proxy:
args.extend(["--proxy", finder.proxy])
for host in finder.trusted_hosts:
args.extend(["--trusted-host", host])
if finder.custom_cert:
args.extend(["--cert", finder.custom_cert])
if finder.client_cert:
args.extend(["--client-cert", finder.client_cert])
if finder.prefer_binary:
args.append("--prefer-binary")
# Handle build constraints
if self._build_constraint_feature_enabled:
args.extend(["--use-feature", "build-constraint"])
if self._build_constraints:
# Build constraints must be passed as both constraints
# and build constraints, so that nested builds receive
# build constraints
for constraint_file in self._build_constraints:
args.extend(["--constraint", constraint_file])
args.extend(["--build-constraint", constraint_file])
extra_environ: ExtraEnviron = {}
if self._build_constraint_feature_enabled and not self._build_constraints:
# If there are no build constraints but the build constraints
# feature is enabled then we must ignore regular constraints
# in the isolated build environment
extra_environ = {"extra_environ": {"_PIP_IN_BUILD_IGNORE_CONSTRAINTS": "1"}}
if finder.uploaded_prior_to:
args.extend(["--uploaded-prior-to", finder.uploaded_prior_to.isoformat()])
args.append("--")
args.extend(requirements)
identify_requirement = (
f" for {for_req.name}" if for_req and for_req.name else ""
)
with open_spinner(f"Installing {kind}") as spinner:
call_subprocess(
args,
command_desc=f"installing {kind}{identify_requirement}",
spinner=spinner,
**extra_environ,
)
class InprocessBuildEnvironmentInstaller:
"""
Build dependency installer that runs in the same pip process.
This contains a stripped down version of the install command with
only the logic necessary for installing build dependencies. The
finder, session, build tracker, and wheel cache are reused, but new
instances of everything else are created as needed.
Options are inherited from the parent install command unless
they don't make sense for build dependencies (in which case, they
are hard-coded, see comments below).
"""
def __init__(
self,
*,
finder: PackageFinder,
build_tracker: BuildTracker,
wheel_cache: WheelCache,
build_constraints: Sequence[InstallRequirement] = (),
verbosity: int = 0,
) -> None:
from pip._internal.operations.prepare import RequirementPreparer
self._finder = finder
self._build_constraints = build_constraints
self._wheel_cache = wheel_cache
self._level = 0
build_dir = TempDirectory(kind="build-env-install", globally_managed=True)
self._preparer = RequirementPreparer(
build_isolation_installer=self,
# Inherited options or state.
finder=finder,
session=finder._link_collector.session,
build_dir=build_dir.path,
build_tracker=build_tracker,
verbosity=verbosity,
# This is irrelevant as it only applies to editable requirements.
src_dir="",
# Hard-coded options (that should NOT be inherited).
download_dir=None,
build_isolation=True,
check_build_deps=False,
progress_bar="off",
# TODO: hash-checking should be extended to build deps, but that is
# deferred for later as it'd be a breaking change.
require_hashes=False,
use_user_site=False,
lazy_wheel=False,
legacy_resolver=False,
)
def install(
self,
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
for_req: InstallRequirement | None,
) -> None:
"""Install entrypoint. Manages output capturing and error handling."""
capture_logs = not logger.isEnabledFor(VERBOSE) and self._level == 0
if capture_logs:
# Hide the logs from the installation of build dependencies.
# They will be shown only if an error occurs.
capture_ctx: ContextManager[StringIO] = capture_logging()
spinner: ContextManager[None] = open_rich_spinner(f"Installing {kind}")
else:
# Otherwise, pass-through all logs (with a header).
capture_ctx, spinner = nullcontext(StringIO()), nullcontext()
logger.info("Installing %s ...", kind)
try:
self._level += 1
with spinner, capture_ctx as stream:
self._install_impl(requirements, prefix)
except DiagnosticPipError as exc:
# Format similar to a nested subprocess error, where the
# causing error is shown first, followed by the build error.
logger.info(textwrap.dedent(stream.getvalue()))
logger.error("%s", exc, extra={"rich": True})
logger.info("")
raise BuildDependencyInstallError(
for_req, requirements, cause=exc, log_lines=None
)
except Exception as exc:
logs: list[str] | None = textwrap.dedent(stream.getvalue()).splitlines()
if not capture_logs:
# If logs aren't being captured, then display the error inline
# with the rest of the logs.
logs = None
if isinstance(exc, PipError):
logger.error("%s", exc)
else:
logger.exception("pip crashed unexpectedly")
raise BuildDependencyInstallError(
for_req, requirements, cause=exc, log_lines=logs
)
finally:
self._level -= 1
def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None:
"""Core build dependency install logic."""
from pip._internal.commands.install import installed_packages_summary
from pip._internal.req import install_given_reqs
from pip._internal.req.constructors import install_req_from_line
from pip._internal.wheel_builder import build
ireqs = [install_req_from_line(req, user_supplied=True) for req in requirements]
ireqs.extend(self._build_constraints)
resolver = self._make_resolver()
resolved_set = resolver.resolve(ireqs, check_supported_wheels=True)
self._preparer.prepare_linked_requirements_more(
resolved_set.requirements.values()
)
reqs_to_build = [
r for r in resolved_set.requirements_to_install if not r.is_wheel
]
_, build_failures = build(reqs_to_build, self._wheel_cache, verify=True)
if build_failures:
raise InstallWheelBuildError(build_failures)
installed = install_given_reqs(
resolver.get_installation_order(resolved_set),
prefix=prefix.path,
# Hard-coded options (that should NOT be inherited).
root=None,
home=None,
warn_script_location=False,
use_user_site=False,
# As the build environment is ephemeral, it's wasteful to
# pre-compile everything since not all modules will be used.
pycompile=False,
progress_bar="off",
)
env = get_environment(list(prefix.lib_dirs))
if summary := installed_packages_summary(installed, env):
logger.info(summary)
def _make_resolver(self) -> BaseResolver:
"""Create a new resolver for one time use."""
# Legacy installer never used the legacy resolver so create a
# resolvelib resolver directly. Yuck.
from pip._internal.req.constructors import install_req_from_req_string
from pip._internal.resolution.resolvelib.resolver import Resolver
return Resolver(
make_install_req=install_req_from_req_string,
# Inherited state.
preparer=self._preparer,
finder=self._finder,
wheel_cache=self._wheel_cache,
# Hard-coded options (that should NOT be inherited).
ignore_requires_python=False,
use_user_site=False,
ignore_dependencies=False,
ignore_installed=True,
force_reinstall=False,
upgrade_strategy="to-satisfy-only",
py_version_info=None,
)
class BuildEnvironment:
"""Creates and manages an isolated environment to install build deps"""
def __init__(self, installer: BuildEnvironmentInstaller) -> None:
self.installer = installer
temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
self._prefixes = OrderedDict(
(name, _Prefix(os.path.join(temp_dir.path, name)))
for name in ("normal", "overlay")
)
self._bin_dirs: list[str] = []
self._lib_dirs: list[str] = []
for prefix in reversed(list(self._prefixes.values())):
self._bin_dirs.append(prefix.bin_dir)
self._lib_dirs.extend(prefix.lib_dirs)
# Customize site to:
# - ensure .pth files are honored
# - prevent access to system site packages
system_sites = _get_system_sitepackages()
self._site_dir = os.path.join(temp_dir.path, "site")
if not os.path.exists(self._site_dir):
os.mkdir(self._site_dir)
with open(
os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
) as fp:
fp.write(
textwrap.dedent(
"""
import os, site, sys
# First, drop system-sites related paths.
original_sys_path = sys.path[:]
known_paths = set()
for path in {system_sites!r}:
site.addsitedir(path, known_paths=known_paths)
system_paths = set(
os.path.normcase(path)
for path in sys.path[len(original_sys_path):]
)
original_sys_path = [
path for path in original_sys_path
if os.path.normcase(path) not in system_paths
]
sys.path = original_sys_path
# Second, add lib directories.
# ensuring .pth file are processed.
for path in {lib_dirs!r}:
assert not path in sys.path
site.addsitedir(path)
"""
).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
)
def __enter__(self) -> None:
self._save_env = {
name: os.environ.get(name, None)
for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
}
path = self._bin_dirs[:]
old_path = self._save_env["PATH"]
if old_path:
path.extend(old_path.split(os.pathsep))
pythonpath = [self._site_dir]
os.environ.update(
{
"PATH": os.pathsep.join(path),
"PYTHONNOUSERSITE": "1",
"PYTHONPATH": os.pathsep.join(pythonpath),
}
)
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> None:
for varname, old_value in self._save_env.items():
if old_value is None:
os.environ.pop(varname, None)
else:
os.environ[varname] = old_value
def check_requirements(
self, reqs: Iterable[str]
) -> tuple[set[tuple[str, str]], set[str]]:
"""Return 2 sets:
- conflicting requirements: set of (installed, wanted) reqs tuples
- missing requirements: set of reqs
"""
missing = set()
conflicting = set()
if reqs:
env = (
get_environment(self._lib_dirs)
if hasattr(self, "_lib_dirs")
else get_default_environment()
)
for req_str in reqs:
req = get_requirement(req_str)
# We're explicitly evaluating with an empty extra value, since build
# environments are not provided any mechanism to select specific extras.
if req.marker is not None and not req.marker.evaluate({"extra": ""}):
continue
dist = env.get_distribution(req.name)
if not dist:
missing.add(req_str)
continue
if isinstance(dist.version, Version):
installed_req_str = f"{req.name}=={dist.version}"
else:
installed_req_str = f"{req.name}==={dist.version}"
if not req.specifier.contains(dist.version, prereleases=True):
conflicting.add((installed_req_str, req_str))
# FIXME: Consider direct URL?
return conflicting, missing
def install_requirements(
self,
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
for_req: InstallRequirement | None = None,
) -> None:
prefix = self._prefixes[prefix_as_string]
assert not prefix.setup
prefix.setup = True
if not requirements:
return
self.installer.install(requirements, prefix, kind=kind, for_req=for_req)
class NoOpBuildEnvironment(BuildEnvironment):
"""A no-op drop-in replacement for BuildEnvironment"""
def __init__(self) -> None:
pass
def __enter__(self) -> None:
pass
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> None:
pass
def cleanup(self) -> None:
pass
def install_requirements(
self,
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
for_req: InstallRequirement | None = None,
) -> None:
raise NotImplementedError()
@@ -16,7 +16,7 @@ from pip._internal.exceptions import InvalidWheelFilename
from pip._internal.models.direct_url import DirectUrl
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryKind
from pip._internal.utils.urls import path_to_url
logger = logging.getLogger(__name__)
@@ -134,10 +134,13 @@ class SimpleWheelCache(Cache):
) -> Link:
candidates = []
if link.is_existing_dir():
return link
if not package_name:
return link
canonical_package_name = canonicalize_name(package_name)
supported_tags_set = set(supported_tags)
for wheel_name, wheel_dir in self._get_candidates(link, canonical_package_name):
try:
wheel = Wheel(wheel_name)
@@ -152,7 +155,7 @@ class SimpleWheelCache(Cache):
package_name,
)
continue
if not wheel.supported(supported_tags):
if not wheel.supported(supported_tags_set):
# Built for a different python/arch/etc
continue
candidates.append(
@@ -175,7 +178,7 @@ class EphemWheelCache(SimpleWheelCache):
def __init__(self) -> None:
self._temp_dir = TempDirectory(
kind=tempdir_kinds.EPHEM_WHEEL_CACHE,
kind=TempDirectoryKind.EPHEM_WHEEL_CACHE,
globally_managed=True,
)
@@ -279,13 +282,16 @@ class WheelCache(Cache):
else:
# TODO: use DirectUrl.equivalent when
# https://github.com/pypa/pip/pull/10564 is merged.
if origin.url != download_info.url:
# origin.url is already stripped because it came from the
# serialized origin.json file.
download_url = download_info.to_dict_compat()["url"]
if origin.url != download_url:
logger.warning(
"Origin URL %s in cache entry %s does not match download URL "
"%s. This is likely a pip bug or a cache corruption issue. "
"Will overwrite it with the new value.",
origin.url,
cache_dir,
download_info.url,
download_url,
)
origin_path.write_text(download_info.to_json(), encoding="utf-8")
@@ -7,11 +7,9 @@ import os
import sys
from collections.abc import Iterable
from itertools import chain
from typing import Any
from pip._internal.cli.main_parser import create_main_parser
from pip._internal.commands import commands_dict, create_command
from pip._internal.metadata import get_default_environment
def autocomplete() -> None:
@@ -51,6 +49,10 @@ def autocomplete() -> None:
"uninstall",
]
if should_list_installed:
# NOTE: this import is deferred until absolutely necessary as it's slow,
# and it's important that autocompletion is fast.
from pip._internal.metadata import get_default_environment
env = get_default_environment()
lc = current.lower()
installed = [
@@ -130,26 +132,33 @@ def autocomplete() -> None:
def get_path_completion_type(
cwords: list[str], cword: int, opts: Iterable[Any]
cwords: list[str], cword: int, opts: Iterable[optparse.Option]
) -> str | None:
"""Get the type of path completion (``file``, ``dir``, ``path`` or None)
:param cwords: same as the environmental variable ``COMP_WORDS``
:param cword: same as the environmental variable ``COMP_CWORD``
:param cwords: Words from ``COMP_WORDS``, excluding the executable
:param cword: Index from ``COMP_CWORD``, including the executable
:param opts: The available options to check
:return: path completion type (``file``, ``dir``, ``path`` or None)
"""
if cword < 2 or not cwords[cword - 2].startswith("-"):
if cword < 2:
return None
# cwords omits the executable, so the previous word is at cword - 2.
previous_option = cwords[cword - 2].split("=", 1)[0]
if not previous_option.startswith("-"):
return None
for opt in opts:
if opt.help == optparse.SUPPRESS_HELP:
if opt.help == optparse.SUPPRESS_HELP or opt.metavar is None:
continue
for o in str(opt).split("/"):
if cwords[cword - 2].split("=")[0] == o:
if not opt.metavar or any(
x in ("path", "file", "dir") for x in opt.metavar.split("/")
):
return opt.metavar
if previous_option not in opt._long_opts + opt._short_opts:
continue
# Metavars such as 'path/url' include a local path alternative.
if any(part in ("path", "file", "dir") for part in opt.metavar.split("/")):
return opt.metavar
return None
@@ -2,14 +2,15 @@
from __future__ import annotations
import contextlib
import logging
import logging.config
import optparse
import os
import sys
import traceback
from collections.abc import Callable, Iterator
from optparse import Values
from typing import Callable
from pip._vendor.rich import reconfigure
from pip._vendor.rich import traceback as rich_traceback
@@ -18,6 +19,7 @@ from pip._internal.cli import cmdoptions
from pip._internal.cli.command_context import CommandContextMixIn
from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip._internal.cli.status_codes import (
BROKEN_STDOUT,
ERROR,
PREVIOUS_BUILD_DIR_ERROR,
UNKNOWN_ERROR,
@@ -50,6 +52,7 @@ class Command(CommandContextMixIn):
def __init__(self, name: str, summary: str, isolated: bool = False) -> None:
super().__init__()
self._cli = False
self.name = name
self.summary = summary
self.parser = ConfigOptionParser(
@@ -80,7 +83,8 @@ class Command(CommandContextMixIn):
def add_options(self) -> None:
pass
def handle_pip_version_check(self, options: Values) -> None:
@contextlib.contextmanager
def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]:
"""
This is a no-op so that commands by default do not do the pip version
check.
@@ -88,16 +92,15 @@ class Command(CommandContextMixIn):
# Make sure we do the pip version check if the index_group options
# are present.
assert not hasattr(options, "no_index")
yield
def run(self, options: Values, args: list[str]) -> int:
raise NotImplementedError
def _run_wrapper(self, level_number: int, options: Values, args: list[str]) -> int:
def _inner_run() -> int:
try:
with self.pip_version_check(options, args):
return self.run(options, args)
finally:
self.handle_pip_version_check(options)
if options.debug_mode:
rich_traceback.install(show_locals=True)
@@ -132,13 +135,28 @@ class Command(CommandContextMixIn):
return ERROR
except BrokenStdoutLoggingError:
# Bypass our logger and write any remaining messages to
# stderr because stdout no longer works.
print("ERROR: Pipe to stdout was broken", file=sys.stderr)
if level_number <= logging.DEBUG:
traceback.print_exc(file=sys.stderr)
# stdout is broken; write to stderr directly. Use os.write, not
# sys.stderr.write, so a full pipe buffer returns EPIPE instead
# of deadlocking (Windows anonymous pipes are ~4KB).
try:
os.write(2, b"ERROR: Pipe to stdout was broken\n")
if level_number <= logging.DEBUG:
encoding = getattr(sys.stderr, "encoding", None) or "utf-8"
os.write(
2, traceback.format_exc().encode(encoding, "backslashreplace")
)
except OSError:
pass
return ERROR
# redirect stdout to os.devnull so that the exit cleanup doesn't
# produce "BrokenPipeError" when exiting.
devnull = os.open(os.devnull, os.O_WRONLY)
try:
os.dup2(devnull, 1)
finally:
os.close(devnull)
return BROKEN_STDOUT
except KeyboardInterrupt:
logger.critical("Operation cancelled by user")
logger.debug("Exception information:", exc_info=True)
@@ -235,17 +253,6 @@ class Command(CommandContextMixIn):
)
options.cache_dir = None
if (
"inprocess-build-deps" in options.features_enabled
and os.environ.get("PIP_CONSTRAINT", "")
and "build-constraint" not in options.features_enabled
):
logger.warning(
"In-process build dependencies are enabled, "
"PIP_CONSTRAINT will have no effect for build dependencies"
)
options.features_enabled.append("build-constraint")
return self._run_wrapper(level_number, options, args)
def handler_map(self) -> dict[str, Callable[[Values, list[str]], None]]:
@@ -14,11 +14,14 @@ from __future__ import annotations
import logging
import os
import pathlib
import re
import textwrap
from collections.abc import Callable
from datetime import datetime, timedelta, timezone
from functools import partial
from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values
from textwrap import dedent
from typing import Any, Callable
from typing import Any
from pip._vendor.packaging.utils import canonicalize_name
@@ -32,6 +35,7 @@ from pip._internal.models.target_python import TargetPython
from pip._internal.utils.datetime import parse_iso_datetime
from pip._internal.utils.hashes import STRONG_HASHES
from pip._internal.utils.misc import strtobool
from pip._internal.utils.urls import strip_extras
logger = logging.getLogger(__name__)
@@ -62,6 +66,37 @@ def make_option_group(group: dict[str, Any], parser: ConfigOptionParser) -> Opti
return option_group
def check_only_deps_option_does_not_conflict(options: Values) -> None:
"""Function for determining if --only-deps and other incompatible options are
specified.
:param options: The OptionParser options.
"""
if not options.only_dependencies:
return
conflicts = []
if options.ignore_dependencies:
conflicts.append("'--no-deps'")
if "legacy-resolver" in options.deprecated_features_enabled:
conflicts.append("'--use-deprecated legacy-resolver'")
if options.requirements:
conflicts.append("'--requirement'")
if options.requirements_from_scripts:
conflicts.append("'--requirements-from-script'")
if options.dependency_groups:
conflicts.append("'--group'")
if conflicts:
if len(conflicts) > 1:
conflicts[-1] = "or " + conflicts[-1]
conflict_message = ", ".join(conflicts)
raise CommandError(
f"Cannot use '--only-dependencies' in combination with {conflict_message}. "
"If this is unexpected, please refer to the user guide:\n"
"\n"
" https://pip.pypa.io/en/stable/user_guide/#installing-only-dependencies"
)
def check_dist_restriction(options: Values, check_target: bool = False) -> None:
"""Function for determining if custom platform options are allowed.
@@ -101,28 +136,21 @@ def check_dist_restriction(options: Values, check_target: bool = False) -> None:
"installing via '--target' or using '--dry-run'"
)
if dist_restriction_set:
# Lazy import to keep CLI startup fast
from pip._internal.utils import pylock as pylock_utils
def check_build_constraints(options: Values) -> None:
"""Function for validating build constraints options.
:param options: The OptionParser options.
"""
if hasattr(options, "build_constraints") and options.build_constraints:
if not options.build_isolation:
raise CommandError(
"--build-constraint cannot be used with --no-build-isolation."
)
# Import here to avoid circular imports
from pip._internal.network.session import PipSession
from pip._internal.req.req_file import get_file_content
# Eagerly check build constraints file contents
# is valid so that we don't fail in when trying
# to check constraints in isolated build process
with PipSession() as session:
for constraint_file in options.build_constraints:
get_file_content(constraint_file, session)
filenames = list(options.requirements)
filenames.extend(path for path, _ in options.dependency_groups)
for filename in filenames:
filename_without_extras, _ = strip_extras(filename)
if pylock_utils.is_valid_pylock_filename(filename_without_extras):
raise CommandError(
"Platform and interpreter constraints using "
"--python-version, --platform, --abi, or --implementation, "
f"are not supported when selecting requirements "
f"from {filename_without_extras!r}"
)
def _path_option_check(option: Option, opt: str, value: str) -> str:
@@ -299,10 +327,19 @@ proxy: Callable[..., Option] = partial(
"--proxy",
dest="proxy",
type="str",
default="",
default=None,
help="Specify a proxy in the form scheme://[user:passwd@]proxy.server:port.",
)
no_proxy_env: Callable[..., Option] = partial(
Option,
"--no-proxy-env",
dest="no_proxy_env",
action="store_true",
default=False,
help="Do not read proxy configuration from environment variables.",
)
retries: Callable[..., Option] = partial(
Option,
"--retries",
@@ -434,8 +471,9 @@ def _handle_uploaded_prior_to(
"""
This is an optparse.Option callback for the --uploaded-prior-to option.
Parses an ISO 8601 datetime string. If no timezone is specified in the string,
local timezone is used.
Accepts either an ISO 8601 datetime string (e.g., '2023-01-01T00:00:00Z')
or a strict subset of ISO 8601 durations: PnD where n is a number of days
(e.g., 'P7D' for 7 days ago).
Note: This option only works with indexes that provide upload-time metadata
as specified in the simple repository API:
@@ -444,6 +482,18 @@ def _handle_uploaded_prior_to(
if value is None:
return None
# Try ISO 8601 duration in PnD format. The leading 'P' disambiguates
# from absolute datetimes. Only whole days are supported; the format may
# be extended to more of the ISO 8601 duration syntax in the future if
# a real need is presented.
match = re.match(r"^P(\d+)D$", value, re.ASCII)
if match:
days = int(match.group(1))
parser.values.uploaded_prior_to = datetime.now(timezone.utc) - timedelta(
days=days
)
return
try:
uploaded_prior_to = parse_iso_datetime(value)
# Use local timezone if no offset is given in the ISO string.
@@ -453,8 +503,9 @@ def _handle_uploaded_prior_to(
except ValueError as exc:
msg = (
f"invalid value: {value!r}: {exc}. "
f"Expected an ISO 8601 datetime string, "
f"e.g '2023-01-01' or '2023-01-01T00:00:00Z'"
f"Expected an ISO 8601 datetime string "
f"(e.g., '2023-01-01' or '2023-01-01T00:00:00Z') "
f"or a duration in days (e.g., 'P3D')"
)
raise_option_error(parser, option=option, msg=msg)
@@ -463,15 +514,18 @@ def uploaded_prior_to() -> Option:
return Option(
"--uploaded-prior-to",
dest="uploaded_prior_to",
metavar="datetime",
metavar="datetime_or_duration",
action="callback",
callback=_handle_uploaded_prior_to,
type="str",
help=(
"Only consider packages uploaded prior to the given date time. "
"Accepts ISO 8601 strings (e.g., '2023-01-01T00:00:00Z'). "
"Uses local timezone if none specified. Only effective when "
"installing from indexes that provide upload-time metadata."
"Only consider packages uploaded prior to the given value. "
"Accepts an ISO 8601 datetime (e.g., '2023-01-01T00:00:00Z', "
"uses local timezone if none specified) "
"or duration in integer days with syntax `P<n>D` "
"(e.g., 'P3D' for packages uploaded at least 3 days ago). "
"Only effective when using indexes that provide "
"upload-time metadata."
),
)
@@ -524,8 +578,15 @@ def requirements() -> Option:
action="append",
default=[],
metavar="file",
help="Install from the given requirements file. "
"This option can be used multiple times.",
help=(
"Install from the given requirements file. "
"The file or URL can be in pip's requirements.txt format, "
"or pylock.toml format. "
"Extras may be selected from a pylock.toml file by "
"appending [extras] to the file name. "
"pylock.toml support is experimental. "
"This option can be used multiple times."
),
)
@@ -536,7 +597,7 @@ def requirements_from_scripts() -> Option:
default=[],
dest="requirements_from_scripts",
metavar="file",
help="Install dependencies of the given script file"
help="Install dependencies of the given script file "
"as defined by PEP 723 inline metadata. ",
)
@@ -615,12 +676,13 @@ def no_binary() -> Option:
callback=_handle_no_binary,
type="str",
default=format_control,
help="Do not use binary packages. Can be supplied multiple times, and "
'each time adds to the existing value. Accepts either ":all:" to '
'disable all binary packages, ":none:" to empty the set (notice '
"the colons), or one or more package names with commas between "
"them (no colons). Note that some packages are tricky to compile "
"and may fail to install when this option is used on them.",
help="Do not download binary packages. Cached binary packages may still "
"be used. Can be supplied multiple times, and each time adds to "
"the existing value. Accepts either ':all:' to disable all binary "
"packages, ':none:' to empty the set (notice the colons), or one "
"or more package names with commas between them (no colons). "
"Note that some packages are tricky to compile and may fail to "
"install when this option is used on them.",
)
@@ -790,15 +852,13 @@ python_version: Callable[..., Option] = partial(
callback=_handle_python_version,
type="str",
default=None,
help=dedent(
"""\
help=dedent("""\
The Python interpreter version to use for wheel and "Requires-Python"
compatibility checks. Defaults to a version derived from the running
interpreter. The version can be specified using up to three dot-separated
integers (e.g. "3" for 3.0.0, "3.7" for 3.7.0, or "3.7.3"). A major-minor
version can also be given as a string without dots (e.g. "37" for 3.7.0).
"""
),
"""),
)
@@ -927,14 +987,67 @@ no_deps: Callable[..., Option] = partial(
)
def _handle_refresh_package(
option: Option, opt_str: str, value: str, parser: OptionParser
) -> None:
if value.startswith("-"):
raise CommandError("--refresh-package option requires 1 argument.")
existing: set[str] = getattr(parser.values, option.dest)
new = value.split(",")
while ":all:" in new:
existing.clear()
existing.add(":all:")
del new[: new.index(":all:") + 1]
if ":none:" not in new:
return
for name in new:
if name == ":none:":
existing.clear()
else:
existing.add(canonicalize_name(name))
def refresh_package() -> Option:
return Option(
"--refresh-package",
dest="refresh_package",
action="callback",
callback=_handle_refresh_package,
type="str",
default=set(),
help="Refresh package index information for the given packages instead "
"of using cached responses. Accepts ':all:' to apply "
"to all packages, or a comma-separated list of package names.",
)
only_deps: Callable[..., Option] = partial(
Option,
"--only-deps",
"--only-dependencies",
dest="only_dependencies",
action="store_true",
default=False,
help=(
"Take only the dependencies of the provided requirements into account, "
"not the requirements themselves. Cannot be used in combination with "
"--no-deps, --group, --requirement, or --requirements-from-script. "
"No user-supplied requirements will be handled, even if they were "
"dependencies of other user-supplied requirements."
),
)
def _handle_dependency_group(
option: Option, opt: str, value: str, parser: OptionParser
) -> None:
"""
Process a value provided for the --group option.
Splits on the rightmost ":", and validates that the path (if present) ends
in `pyproject.toml`. Defaults the path to `pyproject.toml` when one is not given.
Splits on the rightmost ":", and validates an explicit source filename.
`:` cannot appear in dependency group names, so this is a safe and simple parse.
@@ -944,10 +1057,15 @@ def _handle_dependency_group(
if not sep:
path = "pyproject.toml"
else:
# check for 'pyproject.toml' filenames using pathlib
# Local pyproject.toml paths may contain URL-like prefixes.
if pathlib.PurePath(path).name != "pyproject.toml":
msg = "group paths use 'pyproject.toml' filenames"
raise_option_error(parser, option=option, msg=msg)
# Keep lock parsing imports confined to commands that request it.
from pip._internal.utils.pylock import is_valid_pylock_filename
filename, _ = strip_extras(path)
if not is_valid_pylock_filename(filename, allow_urls=False):
msg = "group paths use local 'pyproject.toml' or 'pylock.toml' files"
raise_option_error(parser, option=option, msg=msg)
parser.values.dependency_groups.append((path, groupname))
@@ -961,8 +1079,8 @@ dependency_groups: Callable[..., Option] = partial(
action="callback",
callback=_handle_dependency_group,
metavar="[path:]group",
help='Install a named dependency-group from a "pyproject.toml" file. '
'If a path is given, the name of the file must be "pyproject.toml". '
help='Install a named dependency-group from a "pyproject.toml" or '
'"pylock.toml" file. Named "pylock.<name>.toml" files are also supported. '
'Defaults to using "pyproject.toml" in the current directory.',
)
@@ -1134,6 +1252,17 @@ require_hashes: Callable[..., Option] = partial(
)
no_require_hashes: Callable[..., Option] = partial(
Option,
"--no-require-hashes",
dest="no_require_hashes",
action="store_true",
default=False,
help="Do not automatically enable --require-hashes "
"when encountering a requirement with hashes.",
)
list_path: Callable[..., Option] = partial(
PipOption,
"--path",
@@ -1175,6 +1304,7 @@ no_python_version_warning: Callable[..., Option] = partial(
ALWAYS_ENABLED_FEATURES = [
"truststore", # always on since 24.2
"no-binary-enable-wheel-cache", # always on since 23.1
"build-constraint", # always on since 26.2
]
use_new_feature: Callable[..., Option] = partial(
@@ -1186,8 +1316,8 @@ use_new_feature: Callable[..., Option] = partial(
default=[],
choices=[
"fast-deps",
"build-constraint",
"inprocess-build-deps",
"venv-isolation",
]
+ ALWAYS_ENABLED_FEATURES,
help="Enable new functionality, that may be backward incompatible.",
@@ -1220,12 +1350,12 @@ general_group: dict[str, Any] = {
require_virtualenv,
python,
verbose,
version,
quiet,
log,
no_input,
keyring_provider,
proxy,
no_proxy_env,
retries,
timeout,
exists_action,
@@ -1249,6 +1379,7 @@ index_group: dict[str, Any] = {
index_url,
extra_index_url,
no_index,
refresh_package,
find_links,
uploaded_prior_to,
],
@@ -8,9 +8,10 @@ so commands which don't always hit the network (e.g. list w/o --outdated or
from __future__ import annotations
import contextlib
import logging
import os
import sys
from collections.abc import Iterator
from functools import lru_cache
from optparse import Values
from typing import TYPE_CHECKING
@@ -26,16 +27,13 @@ if TYPE_CHECKING:
from pip._vendor.packaging.utils import NormalizedName
from pip._internal.network.session import PipSession
from pip._internal.self_outdated_check import UpgradePrompt
logger = logging.getLogger(__name__)
@lru_cache
def _create_truststore_ssl_context() -> SSLContext | None:
if sys.version_info < (3, 10):
logger.debug("Disabling truststore because Python version isn't 3.10+")
return None
try:
import ssl
except ImportError:
@@ -102,6 +100,13 @@ class SessionCommandMixin(CommandContextMixIn):
else:
ssl_context = None
has_custom_ca = bool(options.cert)
if options.proxy is None and not options.no_proxy_env:
# Requests reads CA bundle environment variables only with trust_env.
has_custom_ca = has_custom_ca or bool(
os.environ.get("REQUESTS_CA_BUNDLE") or os.environ.get("CURL_CA_BUNDLE")
)
session = PipSession(
cache=os.path.join(cache_dir, "http-v2") if cache_dir else None,
retries=retries if retries is not None else options.retries,
@@ -109,6 +114,8 @@ class SessionCommandMixin(CommandContextMixIn):
trusted_hosts=options.trusted_hosts,
index_urls=self._get_index_urls(options),
ssl_context=ssl_context,
use_ssl_context_for_proxy=not has_custom_ca,
refresh_package=getattr(options, "refresh_package", set()),
)
# Handle custom ca-bundles from the user
@@ -123,14 +130,18 @@ class SessionCommandMixin(CommandContextMixIn):
if options.timeout or timeout:
session.timeout = timeout if timeout is not None else options.timeout
# Handle configured proxies
if options.proxy:
session.proxies = {
"http": options.proxy,
"https": options.proxy,
}
# An explicit --proxy (including "" to disable proxying) or --no-proxy-env
# makes the session ignore environment proxies; unset --proxy is None.
if options.proxy is not None or options.no_proxy_env:
if options.proxy:
session.proxies = {
"http": options.proxy,
"https": options.proxy,
}
session.trust_env = False
# Keep "" so an empty --proxy reaches build subprocesses.
session.pip_proxy = options.proxy
session.pip_no_proxy_env = options.no_proxy_env
# Determine if we can prompt the user for authentication or not
session.auth.prompting = not options.no_input
@@ -139,10 +150,18 @@ class SessionCommandMixin(CommandContextMixIn):
return session
def _pip_self_version_check(session: PipSession, options: Values) -> None:
from pip._internal.self_outdated_check import pip_self_version_check as check
def _pip_self_version_check_fetch(
session: PipSession, options: Values
) -> UpgradePrompt | None:
from pip._internal.self_outdated_check import pip_self_version_check_fetch
check(session, options)
return pip_self_version_check_fetch(session, options)
def _pip_self_version_check_emit(upgrade_prompt: UpgradePrompt | None) -> None:
from pip._internal.self_outdated_check import pip_self_version_check_emit
pip_self_version_check_emit(upgrade_prompt)
class IndexGroupCommand(Command, SessionCommandMixin):
@@ -169,7 +188,8 @@ class IndexGroupCommand(Command, SessionCommandMixin):
# No specific setting: exclude prereleases by default
return True
def handle_pip_version_check(self, options: Values) -> None:
@contextlib.contextmanager
def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]:
"""
Do the pip version check if not disabled.
@@ -179,17 +199,27 @@ class IndexGroupCommand(Command, SessionCommandMixin):
assert hasattr(options, "no_index")
if options.disable_pip_version_check or options.no_index:
yield
return
upgrade_prompt: UpgradePrompt | None = None
try:
# Otherwise, check if we're using the latest version of pip available.
session = self._build_session(
options,
retries=0,
timeout=min(5, options.timeout),
)
with session:
_pip_self_version_check(session, options)
upgrade_prompt = _pip_self_version_check_fetch(session, options)
except Exception:
logger.warning("There was an error checking the latest version of pip.")
logger.debug("See below for error", exc_info=True)
try:
yield
finally:
try:
_pip_self_version_check_emit(upgrade_prompt)
except Exception:
logger.warning("There was an error checking the latest version of pip.")
logger.debug("See below for error", exc_info=True)
@@ -38,7 +38,16 @@ logger = logging.getLogger(__name__)
# main, this should not be an issue in practice.
def cli(args: list[str] | None = None) -> int:
"""Run the CLI with imports blocked until process exit after installation."""
return _main(args, cli=True)
def main(args: list[str] | None = None) -> int:
return _main(args, cli=False)
def _main(args: list[str] | None, *, cli: bool) -> int:
# NOTE: Lazy imports to speed up import of this module,
# which is imported from the pip console script. This doesn't
# speed up normal pip execution, but might be important in the future
@@ -73,13 +82,16 @@ def main(args: list[str] | None = None) -> int:
sys.stderr.write(os.linesep)
sys.exit(1)
# Needed for locale.getpreferredencoding(False) to work
# Needed for locale encoding detection to work
# in pip._internal.utils.encoding.auto_decode
# TODO: Re-evaluate whether this is still needed once pip drops Python 3.10.
try:
locale.setlocale(locale.LC_ALL, "")
except locale.Error as e:
# setlocale can apparently crash if locale are uninitialized
logger.debug("Ignoring error %s when setting locale", e)
command = create_command(cmd_name, isolated=("--isolated" in cmd_args))
command._cli = cli
return command.main(cmd_args)
@@ -8,12 +8,11 @@ import sys
from pip._vendor.rich.markup import escape
from pip._internal.build_env import get_runnable_pip
from pip._internal.cli import cmdoptions
from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip._internal.commands import commands_dict, get_similar_commands
from pip._internal.exceptions import CommandError
from pip._internal.utils.misc import get_pip_version, get_prog
from pip._internal.utils.misc import get_pip_version, get_prog, get_runnable_pip
__all__ = ["create_main_parser", "parse_command"]
@@ -31,6 +30,7 @@ def create_main_parser() -> ConfigOptionParser:
parser.disable_interspersed_args()
parser.version = get_pip_version()
parser.add_option(cmdoptions.version())
# add the general options
gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser)
@@ -97,13 +97,23 @@ def parse_command(args: list[str]) -> tuple[str, list[str]]:
# Set a flag so the child doesn't re-invoke itself, causing
# an infinite loop.
os.environ["_PIP_RUNNING_IN_SUBPROCESS"] = "1"
returncode = 0
subprocess_env = os.environ.copy()
subprocess_env["_PIP_RUNNING_IN_SUBPROCESS"] = "1"
try:
proc = subprocess.run(pip_cmd)
returncode = proc.returncode
# Based on CPython's test-runner interrupt cleanup:
# https://github.com/python/cpython/blob/6c695bfad45ddde73d0d674286800fbe295f0627/Lib/test/libregrtest/main.py#L728-L739
with subprocess.Popen(pip_cmd, env=subprocess_env) as proc:
try:
returncode = proc.wait()
except KeyboardInterrupt:
# Reap before warning or traceback rendering reads replaced files.
proc.kill()
proc.wait()
returncode = 1
except (subprocess.SubprocessError, OSError) as exc:
raise CommandError(f"Failed to run pip under {interpreter}: {exc}")
sys.exit(returncode)
# --version
@@ -11,16 +11,18 @@ import sys
import textwrap
from collections.abc import Generator
from contextlib import suppress
from typing import Any, NoReturn
from typing import TYPE_CHECKING, Any, NoReturn
from pip._vendor.rich.markup import escape
from pip._vendor.rich.theme import Theme
from pip._internal.cli.status_codes import UNKNOWN_ERROR
from pip._internal.configuration import Configuration, ConfigurationError
from pip._internal.utils.logging import PipConsole
from pip._internal.utils.misc import redact_auth_from_url, strtobool
if TYPE_CHECKING:
from pip._internal.utils.logging import PipConsole
logger = logging.getLogger(__name__)
@@ -212,7 +214,6 @@ class ConfigOptionParser(CustomOptionParser):
) -> None:
self.name = name
self.config = Configuration(isolated)
assert self.name
super().__init__(*args, **kwargs)
@@ -220,7 +221,14 @@ class ConfigOptionParser(CustomOptionParser):
try:
return option.check_value(key, val)
except optparse.OptionValueError as exc:
print(f"An error occurred during configuration: {exc}")
logger.error(
"An error occurred during configuration: %s. "
"Check your pip configuration files or "
"environment variables for this value. "
"See https://pip.pypa.io/en/stable/topics/configuration/ "
"for more information.",
exc,
)
sys.exit(3)
def _get_ordered_configuration_items(
@@ -344,7 +352,7 @@ class ConfigOptionParser(CustomOptionParser):
self.print_usage(sys.stderr)
self.exit(UNKNOWN_ERROR, f"{msg}\n")
def print_help(self, file: Any = None) -> None:
def _create_console_for_help(self, file: Any = None) -> PipConsole:
# This is unfortunate but necessary since arguments may have not been
# parsed yet at this point, so detect --no-color manually.
no_color = (
@@ -352,7 +360,19 @@ class ConfigOptionParser(CustomOptionParser):
or bool(strtobool(os.environ.get("PIP_NO_COLOR", "no") or "no"))
or "NO_COLOR" in os.environ
)
console = PipConsole(
# Imported lazily: rich's console stack is only needed when help is
# actually rendered, not when this module is imported for `--version`.
from pip._internal.utils.logging import PipConsole
return PipConsole(
theme=Theme(PrettyHelpFormatter.styles), no_color=no_color, file=file
)
def print_usage(self, file: Any = None) -> None:
if self.usage:
console = self._create_console_for_help(file)
console.print(self.get_usage(), highlight=False)
def print_help(self, file: Any = None) -> None:
console = self._create_console_for_help(file)
console.print(self.format_help().rstrip(), highlight=False)
@@ -2,8 +2,8 @@ from __future__ import annotations
import functools
import sys
from collections.abc import Generator, Iterable, Iterator
from typing import TYPE_CHECKING, Callable, Literal, TypeVar
from collections.abc import Callable, Generator, Iterable, Iterator
from typing import TYPE_CHECKING, Literal, TypeVar
from pip._vendor.rich.progress import (
BarColumn,
@@ -9,12 +9,17 @@ from __future__ import annotations
import logging
import os
from collections.abc import Callable
from dataclasses import dataclass, field
from functools import partial
from itertools import chain
from optparse import Values
from typing import Any, Callable, TypeVar
from pathlib import Path
from typing import Any, TypeVar
from pip._internal.build_env import (
BuildEnvironmentInstaller,
BuildIsolationMode,
InprocessBuildEnvironmentInstaller,
SubprocessBuildEnvironmentInstaller,
)
@@ -39,37 +44,98 @@ from pip._internal.req.constructors import (
install_req_from_editable,
install_req_from_line,
install_req_from_parsed_requirement,
install_req_from_pylock_package,
install_req_from_req_string,
)
from pip._internal.req.pep723 import PEP723Exception, pep723_metadata
from pip._internal.req.req_dependency_group import parse_dependency_groups
from pip._internal.req.req_file import parse_requirements
from pip._internal.req.req_file import get_file_content, parse_requirements
from pip._internal.req.req_install import InstallRequirement
from pip._internal.resolution.base import BaseResolver
from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.pylock import (
is_valid_pylock_filename,
select_from_pylock_path_or_url,
)
from pip._internal.utils.temp_dir import (
TempDirectory,
TempDirectoryKind,
TempDirectoryTypeRegistry,
tempdir_kinds,
)
from pip._internal.utils.urls import strip_extras
logger = logging.getLogger(__name__)
def should_ignore_regular_constraints(options: Values) -> bool:
@dataclass
class _PylockRequest:
"""Accumulate one lock's selectors before evaluating its package markers."""
filename: str
extras: set[str] = field(default_factory=set)
dependency_groups: set[str] = field(default_factory=set)
include_default_groups: bool = False
def _pylock_source_key(filename: str) -> tuple[str, str, str]:
"""Identify a source together with the base used for its relative artifacts."""
if filename.lower().startswith(("http:", "https:", "file:")):
return "url", filename, ""
# Preserve case and parent components because either can change which file opens.
source = str(Path(filename).absolute())
artifact_base = str(Path(os.path.dirname(filename)).absolute())
return "path", source, artifact_base
def _collect_requirement_sources(
requirements: list[str], groups: list[tuple[str, str]]
) -> tuple[list[str | _PylockRequest], list[tuple[str, str]]]:
"""Combine selectors for each lock while retaining separate project inputs."""
pylocks: dict[tuple[str, str, str], _PylockRequest] = {}
requirement_files: list[str | _PylockRequest] = []
project_groups: list[tuple[str, str]] = []
sources = chain(((filename, None) for filename in requirements), groups)
for source, group in sources:
filename, extras = strip_extras(source)
if not is_valid_pylock_filename(filename):
if group is None:
requirement_files.append(source)
else:
project_groups.append((source, group))
continue
key = _pylock_source_key(filename)
if key not in pylocks:
pylocks[key] = _PylockRequest(filename)
requirement_files.append(pylocks[key])
request = pylocks[key]
request.extras.update(extras)
if group is None:
request.include_default_groups = True
else:
request.dependency_groups.add(group)
return requirement_files, project_groups
def should_ignore_regular_constraints() -> bool:
"""
Check if regular constraints should be ignored because
we are in a isolated build process and build constraints
feature is enabled but no build constraints were passed.
Whether this process should ignore the regular constraints it inherits.
The parent pip sets ``_PIP_IN_BUILD_IGNORE_CONSTRAINTS`` whenever it starts
an isolated build in a subprocess, so that regular constraints the build
inherits (such as via ``PIP_CONSTRAINT`` or config files) do not affect it.
Only build constraints apply there.
"""
return os.environ.get("_PIP_IN_BUILD_IGNORE_CONSTRAINTS") == "1"
KEEPABLE_TEMPDIR_TYPES = [
tempdir_kinds.BUILD_ENV,
tempdir_kinds.EPHEM_WHEEL_CACHE,
tempdir_kinds.REQ_BUILD,
TempDirectoryKind.BUILD_ENV,
TempDirectoryKind.EPHEM_WHEEL_CACHE,
TempDirectoryKind.REQ_BUILD,
]
@@ -136,6 +202,20 @@ class RequirementCommand(IndexGroupCommand):
self.cmd_opts.add_option(cmdoptions.dependency_groups())
self.cmd_opts.add_option(cmdoptions.no_clean())
def check_build_constraints(self, options: Values) -> None:
"""Check build constraints with the command's network session."""
if not options.build_constraints:
return
if not options.build_isolation:
raise CommandError(
"--build-constraint cannot be used with --no-build-isolation."
)
session = self.get_default_session(options)
for constraint_file in options.build_constraints:
get_file_content(constraint_file, session)
@staticmethod
def determine_resolver_variant(options: Values) -> str:
"""Determines which resolver should be used, based on the given options."""
@@ -152,6 +232,8 @@ class RequirementCommand(IndexGroupCommand):
build_tracker: BuildTracker,
session: PipSession,
finder: PackageFinder,
*,
allow_editables: bool,
use_user_site: bool,
download_dir: str | None = None,
verbosity: int = 0,
@@ -184,9 +266,6 @@ class RequirementCommand(IndexGroupCommand):
# Handle build constraints
build_constraints = getattr(options, "build_constraints", [])
build_constraint_feature_enabled = (
"build-constraint" in options.features_enabled
)
env_installer: BuildEnvironmentInstaller
if "inprocess-build-deps" in options.features_enabled:
@@ -204,14 +283,26 @@ class RequirementCommand(IndexGroupCommand):
env_installer = SubprocessBuildEnvironmentInstaller(
finder,
build_constraints=build_constraints,
build_constraint_feature_enabled=build_constraint_feature_enabled,
)
if not options.build_isolation:
build_isolation: BuildIsolationMode = "off"
elif "venv-isolation" in options.features_enabled:
build_isolation = "venv"
if "inprocess-build-deps" in options.features_enabled:
logger.warning(
"Using --use-feature 'inprocess-build-deps' and 'venv-isolation'"
" together is currently partially supported. If you encounter"
" broken behaviour, use only one feature."
)
else:
build_isolation = "virtual"
return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
download_dir=download_dir,
build_isolation=options.build_isolation,
build_isolation=build_isolation,
build_isolation_installer=env_installer,
check_build_deps=options.check_build_deps,
build_tracker=build_tracker,
@@ -223,6 +314,7 @@ class RequirementCommand(IndexGroupCommand):
lazy_wheel=lazy_wheel,
verbosity=verbosity,
legacy_resolver=legacy_resolver,
allow_editables=allow_editables,
)
@classmethod
@@ -260,6 +352,7 @@ class RequirementCommand(IndexGroupCommand):
make_install_req=make_install_req,
use_user_site=use_user_site,
ignore_dependencies=options.ignore_dependencies,
only_dependencies=options.only_dependencies,
ignore_installed=ignore_installed,
ignore_requires_python=ignore_requires_python,
force_reinstall=force_reinstall,
@@ -275,6 +368,7 @@ class RequirementCommand(IndexGroupCommand):
make_install_req=make_install_req,
use_user_site=use_user_site,
ignore_dependencies=options.ignore_dependencies,
only_dependencies=options.only_dependencies,
ignore_installed=ignore_installed,
ignore_requires_python=ignore_requires_python,
force_reinstall=force_reinstall,
@@ -293,12 +387,19 @@ class RequirementCommand(IndexGroupCommand):
Parse command-line arguments into the corresponding requirements.
"""
requirements: list[InstallRequirement] = []
requirement_sources, project_groups = _collect_requirement_sources(
options.requirements, options.dependency_groups
)
if not should_ignore_regular_constraints(options):
constraints = parse_constraint_files(
options.constraints, finder, options, session
)
requirements.extend(constraints)
if should_ignore_regular_constraints():
# Inside an isolated build subprocess: apply the build constraints
# (forwarded via --build-constraint) instead of the inherited ones.
constraint_files = getattr(options, "build_constraints", [])
else:
constraint_files = options.constraints
constraints = parse_constraint_files(constraint_files, finder, options, session)
requirements.extend(constraints)
for req in args:
if not req.strip():
@@ -312,8 +413,8 @@ class RequirementCommand(IndexGroupCommand):
)
requirements.append(req_to_add)
if options.dependency_groups:
for req in parse_dependency_groups(options.dependency_groups):
if project_groups:
for req in parse_dependency_groups(project_groups):
req_to_add = install_req_from_req_string(
req,
isolated=options.isolated_mode,
@@ -331,9 +432,34 @@ class RequirementCommand(IndexGroupCommand):
requirements.append(req_to_add)
# NOTE: options.require_hashes may be set if --require-hashes is True
for filename in options.requirements:
for source in requirement_sources:
if isinstance(source, _PylockRequest):
logger.warning(
"Using pylock.toml as a requirements source "
"is an experimental feature. "
"It may be removed/changed in a future release "
"without prior warning."
)
for package, package_dist in select_from_pylock_path_or_url(
source.filename,
session=session,
format_control=options.format_control,
extras=source.extras,
dependency_groups=source.dependency_groups,
include_default_groups=source.include_default_groups,
):
req_to_add, locked_link = install_req_from_pylock_package(
package,
package_dist,
source.filename,
user_supplied=True,
)
requirements.append(req_to_add)
if locked_link:
finder.add_locked_link(package.name, locked_link)
continue
for parsed_req in parse_requirements(
filename, finder=finder, options=options, session=session
source, finder=finder, options=options, session=session
):
req_to_add = install_req_from_parsed_requirement(
parsed_req,
@@ -379,8 +505,17 @@ class RequirementCommand(IndexGroupCommand):
)
requirements.append(req_to_add)
# If any requirement has hash options, enable hash checking.
if any(req.has_hash_options for req in requirements):
if options.require_hashes and options.no_require_hashes:
raise CommandError(
"--require-hashes and --no-require-hashes are mutually exclusive"
)
# If any requirement has hash options, enable hash checking for all
# requirements, unless this mechanism has been explicitly disabled
# with --no-require-hashes.
if not options.no_require_hashes and any(
req.has_hash_options for req in requirements
):
options.require_hashes = True
if not (
@@ -422,7 +557,7 @@ class RequirementCommand(IndexGroupCommand):
options: Values,
session: PipSession,
target_python: TargetPython | None = None,
ignore_requires_python: bool | None = None,
ignore_requires_python: bool = False,
) -> PackageFinder:
"""
Create a package finder appropriate to this requirement command.
@@ -4,3 +4,4 @@ UNKNOWN_ERROR = 2
VIRTUALENV_NOT_FOUND = 3
PREVIOUS_BUILD_DIR_ERROR = 4
NO_MATCHES_FOUND = 23
BROKEN_STDOUT = 120
@@ -5,12 +5,17 @@ Package containing all pip commands
from __future__ import annotations
import importlib
from collections import namedtuple
from typing import Any
from typing import TYPE_CHECKING, Any, NamedTuple
from pip._internal.cli.base_command import Command
if TYPE_CHECKING:
from pip._internal.cli.base_command import Command
class CommandInfo(NamedTuple):
module_path: str
class_name: str
summary: str
CommandInfo = namedtuple("CommandInfo", "module_path, class_name, summary")
# This dictionary does a bunch of heavy lifting for help output:
# - Enables avoiding additional (costly) imports for presenting `--help`.
@@ -119,7 +124,9 @@ def create_command(name: str, **kwargs: Any) -> Command:
"""
module_path, class_name, summary = commands_dict[name]
module = importlib.import_module(module_path)
command_class = getattr(module, class_name)
# Registry entries identify Command subclasses by module and class name.
command_class: type[Command] = getattr(module, class_name)
command = command_class(name=name, summary=summary, **kwargs)
return command
@@ -1,7 +1,7 @@
import os
import textwrap
from collections.abc import Callable
from optparse import Values
from typing import Callable
from pip._internal.cli.base_command import Command
from pip._internal.cli.status_codes import ERROR, SUCCESS
@@ -107,8 +107,7 @@ class CacheCommand(Command):
wheels_cache_size = filesystem.format_directory_size(wheels_cache_location)
message = (
textwrap.dedent(
"""
textwrap.dedent("""
Package index page cache location (pip v23.3+): {http_cache_location}
Package index page cache location (older pips): {old_http_cache_location}
Package index page cache size: {http_cache_size}
@@ -116,8 +115,7 @@ class CacheCommand(Command):
Locally built wheels location: {wheels_cache_location}
Locally built wheels size: {wheels_cache_size}
Number of locally built wheels: {package_count}
""" # noqa: E501
)
""") # noqa: E501
.format(
http_cache_location=http_cache_location,
old_http_cache_location=old_http_cache_location,
@@ -191,10 +189,13 @@ class CacheCommand(Command):
logger.verbose("Removed %s", filename)
http_dirs = filesystem.subdirs_without_files(self._cache_dir(options, "http"))
http_v2_dirs = filesystem.subdirs_without_files(
self._cache_dir(options, "http-v2")
)
wheel_dirs = filesystem.subdirs_without_wheels(
self._cache_dir(options, "wheels")
)
dirs = [*http_dirs, *wheel_dirs]
dirs = [*http_dirs, *http_v2_dirs, *wheel_dirs]
for subdir in dirs:
try:
@@ -1,10 +1,11 @@
from __future__ import annotations
import inspect
import logging
import os
import subprocess
from collections.abc import Callable
from optparse import Values
from typing import Any, Callable
from pip._internal.cli.base_command import Command
from pip._internal.cli.status_codes import ERROR, SUCCESS
@@ -12,7 +13,6 @@ from pip._internal.configuration import (
Configuration,
Kind,
get_configuration_files,
kinds,
)
from pip._internal.exceptions import PipError
from pip._internal.utils.logging import indent_log
@@ -21,6 +21,30 @@ from pip._internal.utils.misc import get_prog, write_output
logger = logging.getLogger(__name__)
def _config_handler(
name: str, handler: Callable[..., None]
) -> Callable[[Values, list[str]], None]:
"""Adapt a configuration action's positional parameters to CLI arguments."""
signature = inspect.signature(handler)
def invoke(options: Values, args: list[str]) -> None:
try:
signature.bind(options, *args)
except TypeError:
# Skip options; these action handlers are bound methods.
arguments = list(signature.parameters)[1:]
example = " ".join([name, *(f"[{argument}]" for argument in arguments)])
raise PipError(
f"Got unexpected number of arguments, expected {len(arguments)}. "
f'(example: "{get_prog()} config {example}")'
) from None
# TypeErrors raised by the action itself are programming errors.
handler(options, *args)
return invoke
class ConfigurationCommand(Command):
"""
Manage local and global configuration.
@@ -96,7 +120,7 @@ class ConfigurationCommand(Command):
self.parser.insert_option_group(0, self.cmd_opts)
def handler_map(self) -> dict[str, Callable[[Values, list[str]], None]]:
return {
handlers: dict[str, Callable[..., None]] = {
"list": self.list_values,
"edit": self.open_in_editor,
"get": self.get_name,
@@ -105,6 +129,10 @@ class ConfigurationCommand(Command):
"debug": self.list_config_values,
}
return {
name: _config_handler(name, handler) for name, handler in handlers.items()
}
def run(self, options: Values, args: list[str]) -> int:
handler_map = self.handler_map()
@@ -147,9 +175,9 @@ class ConfigurationCommand(Command):
file_options = [
key
for key, value in (
(kinds.USER, options.user_file),
(kinds.GLOBAL, options.global_file),
(kinds.SITE, options.site_file),
(Kind.USER, options.user_file),
(Kind.GLOBAL, options.global_file),
(Kind.SITE, options.site_file),
)
if value
]
@@ -160,11 +188,11 @@ class ConfigurationCommand(Command):
# Default to user, unless there's a site file.
elif any(
os.path.exists(site_config_file)
for site_config_file in get_configuration_files()[kinds.SITE]
for site_config_file in get_configuration_files()[Kind.SITE]
):
return kinds.SITE
return Kind.SITE
else:
return kinds.USER
return Kind.USER
elif len(file_options) == 1:
return file_options[0]
@@ -173,34 +201,28 @@ class ConfigurationCommand(Command):
"(--user, --site, --global) to perform."
)
def list_values(self, options: Values, args: list[str]) -> None:
self._get_n_args(args, "list", n=0)
def list_values(self, options: Values) -> None:
for key, value in sorted(self.configuration.items()):
for key, value in sorted(value.items()):
write_output("%s=%r", key, value)
def get_name(self, options: Values, args: list[str]) -> None:
key = self._get_n_args(args, "get [name]", n=1)
value = self.configuration.get_value(key)
def get_name(self, options: Values, name: str) -> None:
value = self.configuration.get_value(name)
write_output("%s", value)
def set_name_value(self, options: Values, args: list[str]) -> None:
key, value = self._get_n_args(args, "set [name] [value]", n=2)
self.configuration.set_value(key, value)
def set_name_value(self, options: Values, name: str, value: str) -> None:
self.configuration.set_value(name, value)
self._save_configuration()
def unset_name(self, options: Values, args: list[str]) -> None:
key = self._get_n_args(args, "unset [name]", n=1)
self.configuration.unset_value(key)
def unset_name(self, options: Values, name: str) -> None:
self.configuration.unset_value(name)
self._save_configuration()
def list_config_values(self, options: Values, args: list[str]) -> None:
def list_config_values(self, options: Values) -> None:
"""List config key-value pairs across different config files"""
self._get_n_args(args, "debug", n=0)
self.print_env_var_values()
# Iterate over config files and print if they exist, and the
@@ -230,7 +252,7 @@ class ConfigurationCommand(Command):
env_var = f"PIP_{key.upper()}"
write_output("%s=%r", env_var, value)
def open_in_editor(self, options: Values, args: list[str]) -> None:
def open_in_editor(self, options: Values) -> None:
editor = self._determine_editor(options)
fname = self.configuration.get_file_to_edit()
@@ -252,20 +274,6 @@ class ConfigurationCommand(Command):
except subprocess.CalledProcessError as e:
raise PipError(f"Editor Subprocess exited with exit code {e.returncode}")
def _get_n_args(self, args: list[str], example: str, n: int) -> Any:
"""Helper to make sure the command got the right number of arguments"""
if len(args) != n:
msg = (
f"Got unexpected number of arguments, expected {n}. "
f'(example: "{get_prog()} config {example}")'
)
raise PipError(msg)
if n == 1:
return args[0]
else:
return args
def _save_configuration(self) -> None:
# We successfully ran a modifying command. Need to save the
# configuration.
@@ -1,6 +1,6 @@
from __future__ import annotations
import locale
import importlib
import logging
import os
import sys
@@ -18,7 +18,7 @@ from pip._internal.cli.cmdoptions import make_target_python
from pip._internal.cli.status_codes import SUCCESS
from pip._internal.configuration import Configuration
from pip._internal.metadata import get_environment
from pip._internal.utils.compat import open_text_resource
from pip._internal.utils.compat import get_locale_encoding, open_text_resource
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import get_pip_version
@@ -48,29 +48,21 @@ def create_vendor_txt_map() -> dict[str, str]:
return dict(line.split("==", 1) for line in lines)
def get_module_from_module_name(module_name: str) -> ModuleType | None:
def get_module_from_module_name(module_name: str) -> ModuleType:
# Module name can be uppercase in vendor.txt for some reason...
module_name = module_name.lower().replace("-", "_")
# PATCH: setuptools is actually only pkg_resources.
if module_name == "setuptools":
module_name = "pkg_resources"
try:
__import__(f"pip._vendor.{module_name}", globals(), locals(), level=0)
return getattr(pip._vendor, module_name)
except ImportError:
# We allow 'truststore' to fail to import due
# to being unavailable on Python 3.9 and earlier.
if module_name == "truststore" and sys.version_info < (3, 10):
return None
raise
return importlib.import_module(f"pip._vendor.{module_name}")
def get_vendor_version_from_module(module_name: str) -> str | None:
module = get_module_from_module_name(module_name)
version = getattr(module, "__version__", None)
if module and not version:
if not version:
# Try to find version in debundled module info.
assert module.__file__ is not None
env = get_environment([os.path.dirname(module.__file__)])
@@ -141,7 +133,7 @@ def show_tags(options: Values) -> None:
def ca_bundle_info(config: Configuration) -> str:
levels = {key.split(".", 1)[0] for key, _ in config.items()}
levels = {key.split(".", 1)[0] for _, options in config.items() for key in options}
if not levels:
return "Not specified"
@@ -184,8 +176,8 @@ class DebugCommand(Command):
show_value("sys.getdefaultencoding", sys.getdefaultencoding())
show_value("sys.getfilesystemencoding", sys.getfilesystemencoding())
show_value(
"locale.getpreferredencoding",
locale.getpreferredencoding(),
"locale.getencoding",
get_locale_encoding(),
)
show_value("sys.platform", sys.platform)
show_sys_implementation()
@@ -39,8 +39,10 @@ class DownloadCommand(RequirementCommand):
self.cmd_opts.add_option(cmdoptions.requirements())
self.cmd_opts.add_option(cmdoptions.requirements_from_scripts())
self.cmd_opts.add_option(cmdoptions.no_deps())
self.cmd_opts.add_option(cmdoptions.only_deps())
self.cmd_opts.add_option(cmdoptions.src())
self.cmd_opts.add_option(cmdoptions.require_hashes())
self.cmd_opts.add_option(cmdoptions.no_require_hashes())
self.cmd_opts.add_option(cmdoptions.progress_bar())
self.cmd_opts.add_option(cmdoptions.no_build_isolation())
self.cmd_opts.add_option(cmdoptions.use_pep517())
@@ -82,8 +84,9 @@ class DownloadCommand(RequirementCommand):
options.editables = []
cmdoptions.check_dist_restriction(options)
cmdoptions.check_build_constraints(options)
self.check_build_constraints(options)
cmdoptions.check_release_control_exclusive(options)
cmdoptions.check_only_deps_option_does_not_conflict(options)
options.download_dir = normalize_path(options.download_dir)
ensure_dir(options.download_dir)
@@ -117,6 +120,7 @@ class DownloadCommand(RequirementCommand):
download_dir=options.download_dir,
use_user_site=False,
verbosity=self.verbosity,
allow_editables=False,
)
resolver = self.make_resolver(
@@ -4,8 +4,8 @@ from optparse import Values
from pip._internal.cli import cmdoptions
from pip._internal.cli.base_command import Command
from pip._internal.cli.status_codes import SUCCESS
from pip._internal.metadata.base import stdlib_pkgs
from pip._internal.operations.freeze import freeze
from pip._internal.utils.compat import stdlib_pkgs
def _should_suppress_build_backends() -> bool:
@@ -2,9 +2,9 @@ from __future__ import annotations
import json
import logging
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from optparse import Values
from typing import Any, Callable
from typing import Any
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
@@ -16,7 +16,12 @@ from pip._internal.commands.search import (
get_installed_distribution,
print_dist_installation_info,
)
from pip._internal.exceptions import CommandError, DistributionNotFound, PipError
from pip._internal.exceptions import (
CommandError,
DiagnosticPipError,
DistributionNotFound,
PipError,
)
from pip._internal.index.collector import LinkCollector
from pip._internal.index.package_finder import PackageFinder
from pip._internal.models.selection_prefs import SelectionPreferences
@@ -80,6 +85,8 @@ class IndexCommand(IndexGroupCommand):
# Error handling happens here, not in the action-handlers.
try:
handler_map[action](options, args[1:])
except DiagnosticPipError:
raise
except PipError as e:
logger.error(e.args[0])
return ERROR
@@ -91,7 +98,7 @@ class IndexCommand(IndexGroupCommand):
options: Values,
session: PipSession,
target_python: TargetPython | None = None,
ignore_requires_python: bool | None = None,
ignore_requires_python: bool = False,
) -> PackageFinder:
"""
Create a package finder appropriate to the index command.
@@ -10,7 +10,7 @@ from pip._internal.cli import cmdoptions
from pip._internal.cli.base_command import Command
from pip._internal.cli.status_codes import SUCCESS
from pip._internal.metadata import BaseDistribution, get_environment
from pip._internal.utils.compat import stdlib_pkgs
from pip._internal.metadata.base import stdlib_pkgs
from pip._internal.utils.urls import path_to_url
logger = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ class InspectCommand(Command):
# report) since it is not recorded in installed metadata.
direct_url = dist.direct_url
if direct_url is not None:
res["direct_url"] = direct_url.to_dict()
res["direct_url"] = direct_url.to_dict_compat()
else:
# Emulate direct_url for legacy editable installs.
editable_project_location = dist.editable_project_location
@@ -1,17 +1,29 @@
from __future__ import annotations
import contextlib
import errno
import functools
import inspect
import json
import operator
import os
import shutil
import site
import sys
import traceback
import types
from collections.abc import Iterable, Iterator, Sequence
from optparse import SUPPRESS_HELP, Values
from pathlib import Path
from typing import Any
from pip._vendor import pygments, rich
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.requests.exceptions import InvalidProxyURL
from pip._vendor.rich import print_json
from pip._vendor.rich.text import Text
import pip
# Eagerly import self_outdated_check to avoid crashes. Otherwise,
# this module would be imported *after* pip was replaced, resulting
@@ -19,7 +31,7 @@ from pip._vendor.rich import print_json
# with the rest of pip that's already imported, or allowing a
# wheel to execute arbitrary code on install by replacing
# self_outdated_check.
import pip._internal.self_outdated_check # noqa: F401
import pip._internal.self_outdated_check
from pip._internal.cache import WheelCache
from pip._internal.cli import cmdoptions
from pip._internal.cli.cmdoptions import make_target_python
@@ -29,6 +41,7 @@ from pip._internal.cli.req_command import (
)
from pip._internal.cli.status_codes import ERROR, SUCCESS
from pip._internal.exceptions import (
BlockedImportError,
CommandError,
InstallationError,
InstallWheelBuildError,
@@ -63,6 +76,242 @@ from pip._internal.wheel_builder import build
logger = getLogger(__name__)
_IMPORT_AUDIT_HOOK_INSTALLED = False
_ACTIVE_INSTALLATIONS: set[object] = set()
_WARNED_IMPORTS: set[str] = set()
_IMPORT_ERRORS: dict[str, ImportError] = {}
# Import dependencies directly where possible; otherwise list them here.
_INTERNAL_EAGER_IMPORTS: tuple[str, ...] = (
"pip._internal.operations.install.wheel",
# Rich loads the Python lexer when rendering debug tracebacks.
f"{pygments.__name__}.lexers.python",
# Preload Python 3.13+ traceback helpers to preserve the original error.
"traceback",
"_colorize",
"_suggestions",
"ast",
"difflib",
"unicodedata",
# Preserve cmd's fallback when readline is unavailable.
"readline",
# weakref.finalize imports gc when running shutdown callbacks.
"gc",
# Requests imports netrc lazily when looking up credentials.
"netrc",
# Use Rich's actual namespace when pip has been de-vendored.
# The renderer is used for output to a legacy Windows console.
f"{rich.__name__}._windows_renderer",
# zipfile may import these backends when reading compressed wheel contents.
"_bz2",
"_lzma",
"_zstd",
)
def _warn_unsafe_import(module: str) -> None:
"""Warn once per module without letting logging block the import."""
if module in _WARNED_IMPORTS:
return
# Mark before logging: a handler may itself import this module.
_WARNED_IMPORTS.add(module)
try:
logger.warning(
"Unsafe import of %r after pip install completed: packages may have "
"changed. Callers should import all required modules before running "
"pip install, or run pip in a subprocess. "
"See https://pip.pypa.io/en/stable/user_guide/"
"#using-pip-from-your-program",
module,
)
except Exception:
# The caller's logging handlers must not turn warnings into import errors.
pass
def _prevent_import_hook(name: str, args: tuple[Any, ...]) -> None:
if name == "import":
_check_import(args[0])
def _check_import(module: str) -> None:
"""Apply the installation import policy to audit events and module lookups."""
if not _ACTIVE_INSTALLATIONS:
_warn_unsafe_import(module)
return
if module in _IMPORT_ERRORS:
raise _IMPORT_ERRORS[module].with_traceback(None)
raise BlockedImportError(
f"Blocked import of {module!r} after pip install started. Hook authors"
" must import required modules before installation starts."
)
class _ImportBlocker:
"""Cover module lookups, including reloads, without import audit events."""
def find_spec(
self,
fullname: str,
path: Sequence[str] | None = None,
target: types.ModuleType | None = None,
) -> None:
_check_import(fullname)
_IMPORT_BLOCKER = _ImportBlocker()
def _eagerly_import_modules(module_names: Iterable[str]) -> None:
"""Import selected dependencies before installation starts."""
# Load Rich's lazy emoji and width data before installation starts.
_ = Text.from_markup(":package:").cell_len
_IMPORT_ERRORS.clear()
for module in dict.fromkeys(module_names):
try:
__import__(module)
except ImportError as error:
_IMPORT_ERRORS[module] = error.with_traceback(None)
def _referenced_modules(module: types.ModuleType) -> Iterator[types.ModuleType]:
"""Find dependencies referenced by modules and callables."""
for value in tuple(vars(module).values()):
if isinstance(value, types.ModuleType):
yield value
continue
# Standard callable adapters keep dependency globals on their function.
while True:
if type(value) is functools.partial:
value = value.func
elif type(value) is types.MethodType:
value = value.__func__
else:
break
names: tuple[object, ...]
if type(value) is types.FunctionType:
# Decorators can preserve the original module while changing globals.
names = (value.__globals__.get("__name__"), value.__module__)
elif callable(value):
# Read callable metadata without invoking descriptors.
names = (inspect.getattr_static(value, "__module__", None),)
else:
continue
for name in names:
dependency = sys.modules.get(name) if isinstance(name, str) else None
if isinstance(dependency, types.ModuleType):
yield dependency
def _cache_lazy_import_error(error: ImportError) -> None:
"""Preserve failures for both a missing dependency and its failed importers."""
# Import machinery removes failed modules before propagating their error.
for frame, _ in traceback.walk_tb(error.__traceback__):
module_name = frame.f_globals.get("__name__")
if isinstance(module_name, str) and module_name not in sys.modules:
_IMPORT_ERRORS[module_name] = error
if error.name is not None:
_IMPORT_ERRORS[error.name] = error
# Avoid retaining the import traceback in the cache.
error.with_traceback(None)
def _reify_lazy_imports(module_names: Iterable[str]) -> None:
"""Resolve lazy bindings in installation dependencies before imports are blocked."""
if sys.version_info < (3, 15):
return
modules = {
name: module
for name in module_names
if isinstance(module := sys.modules.get(name), types.ModuleType)
}
attempted: set[object] = set()
while True:
# A from-import can already hold an object from a lazy dependency.
dependencies = {
dependency.__name__: dependency
for module in modules.values()
for dependency in _referenced_modules(module)
if dependency.__name__ not in modules
}
modules.update(dependencies)
# Resolving a binding can add lazy imports to modules already scanned.
pending = [
(module, name, value)
for module in modules.values()
for name, value in tuple(vars(module).items())
if type(value) is types.LazyImportType and value not in attempted
]
if not pending and not dependencies:
break
for module, name, value in pending:
attempted.add(value)
before = set(sys.modules)
try:
resolved = getattr(module, name)
# The binding may resolve to an already-loaded module.
if isinstance(resolved, types.ModuleType):
modules[resolved.__name__] = resolved
except ImportError as error:
_cache_lazy_import_error(error)
except Exception:
# Optional lazy imports need not succeed for installation to proceed.
pass
# Include dependencies loaded indirectly, even by a failed import.
for dependency in sys.modules.keys() - before:
imported = sys.modules[dependency]
if isinstance(imported, types.ModuleType):
modules[dependency] = imported
@contextlib.contextmanager
def _prevent_further_imports(*, until_process_exit: bool = False) -> Iterator[None]:
"""Block late imports until process exit or control returns to a caller."""
global _IMPORT_AUDIT_HOOK_INSTALLED
if not _IMPORT_AUDIT_HOOK_INSTALLED:
sys.addaudithook(_prevent_import_hook)
_IMPORT_AUDIT_HOOK_INSTALLED = True
# Callers may add finders between installations; reloads bypass audit events.
sys.meta_path[:] = [
_IMPORT_BLOCKER,
*(finder for finder in sys.meta_path if finder is not _IMPORT_BLOCKER),
]
token = object()
_ACTIVE_INSTALLATIONS.add(token)
try:
yield
finally:
# CLI exception hooks and exit callbacks run after command cleanup.
if not until_process_exit:
_ACTIVE_INSTALLATIONS.discard(token)
def _arg_refers_to_pip(arg: str) -> bool:
try:
req = Requirement(arg)
except InvalidRequirement:
return False
return canonicalize_name(req.name) == "pip"
class InstallCommand(RequirementCommand):
"""
Install packages from:
@@ -83,12 +332,18 @@ class InstallCommand(RequirementCommand):
%prog [options] [-e] <local project path> ...
%prog [options] <archive url/path> ..."""
def main(self, args: list[str]) -> int:
# Keep imports blocked through command cleanup and logging shutdown.
with contextlib.ExitStack() as self._import_guard:
return super().main(args)
def add_options(self) -> None:
self.cmd_opts.add_option(cmdoptions.requirements())
self.cmd_opts.add_option(cmdoptions.constraints())
self.cmd_opts.add_option(cmdoptions.build_constraints())
self.cmd_opts.add_option(cmdoptions.requirements_from_scripts())
self.cmd_opts.add_option(cmdoptions.no_deps())
self.cmd_opts.add_option(cmdoptions.only_deps())
self.cmd_opts.add_option(cmdoptions.editable())
self.cmd_opts.add_option(
@@ -245,6 +500,7 @@ class InstallCommand(RequirementCommand):
help="Do not warn about broken dependencies",
)
self.cmd_opts.add_option(cmdoptions.require_hashes())
self.cmd_opts.add_option(cmdoptions.no_require_hashes())
self.cmd_opts.add_option(cmdoptions.progress_bar())
self.cmd_opts.add_option(cmdoptions.root_user_action())
@@ -278,6 +534,17 @@ class InstallCommand(RequirementCommand):
),
)
@contextlib.contextmanager
def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]:
# Skip the self-version check when pip itself is a requirement. The
# running pip may be replaced mid-command, and the upgrade prompt
# is redundant.
if any(_arg_refers_to_pip(arg) for arg in args):
yield
return
with super().pip_version_check(options, args):
yield
@with_cleanup
def run(self, options: Values, args: list[str]) -> int:
if options.use_user_site and options.target_dir is not None:
@@ -304,7 +571,8 @@ class InstallCommand(RequirementCommand):
if options.upgrade:
upgrade_strategy = options.upgrade_strategy
cmdoptions.check_build_constraints(options)
cmdoptions.check_only_deps_option_does_not_conflict(options)
self.check_build_constraints(options)
cmdoptions.check_dist_restriction(options, check_target=True)
cmdoptions.check_release_control_exclusive(options)
@@ -359,12 +627,6 @@ class InstallCommand(RequirementCommand):
wheel_cache = WheelCache(options.cache_dir)
# Only when installing is it permitted to use PEP 660.
# In other circumstances (pip wheel, pip download) we generate
# regular (i.e. non editable) metadata and wheels.
for req in reqs:
req.permit_editable_wheels = True
preparer = self.make_requirement_preparer(
temp_build_dir=directory,
options=options,
@@ -373,6 +635,7 @@ class InstallCommand(RequirementCommand):
finder=finder,
use_user_site=options.use_user_site,
verbosity=self.verbosity,
allow_editables=True,
)
resolver = self.make_resolver(
preparer=preparer,
@@ -396,7 +659,7 @@ class InstallCommand(RequirementCommand):
if options.json_report_file:
report = InstallationReport(requirement_set.requirements_to_install)
if options.json_report_file == "-":
print_json(data=report.to_dict())
rich.print_json(data=report.to_dict())
else:
with open(options.json_report_file, "w", encoding="utf-8") as f:
json.dump(report.to_dict(), f, indent=2, ensure_ascii=False)
@@ -438,6 +701,7 @@ class InstallCommand(RequirementCommand):
reqs_to_build,
wheel_cache=wheel_cache,
verify=True,
allow_editables=True,
)
if build_failures:
@@ -459,6 +723,15 @@ class InstallCommand(RequirementCommand):
if options.target_dir or options.prefix_path:
warn_script_location = False
# Block late imports so we don't silently pick up a module
# from a distribution pip is about to install.
module_names = (*_INTERNAL_EAGER_IMPORTS, *pip._EAGER_IMPORTS)
_eagerly_import_modules(module_names)
_reify_lazy_imports(module_names)
self._import_guard.enter_context(
_prevent_further_imports(until_process_exit=self._cli)
)
installed = install_given_reqs(
to_install,
root=options.root_path,
@@ -503,8 +776,10 @@ class InstallCommand(RequirementCommand):
self._handle_target_dir(
options.target_dir, target_temp_dir, options.upgrade
)
if options.root_user_action == "warn":
warn_if_run_as_root()
return SUCCESS
def _handle_target_dir(
@@ -512,8 +787,6 @@ class InstallCommand(RequirementCommand):
) -> None:
ensure_dir(target_dir)
# Checking both purelib and platlib directories for installed
# packages to be moved to target directory
lib_dir_list = []
# Checking both purelib and platlib directories for installed
@@ -532,12 +805,16 @@ class InstallCommand(RequirementCommand):
for lib_dir in lib_dir_list:
for item in os.listdir(lib_dir):
# Do not move library directories a second time as data.
if lib_dir == data_dir:
ddir = os.path.join(data_dir, item)
if any(s.startswith(ddir) for s in lib_dir_list[:-1]):
continue
target_item_dir = os.path.join(target_dir, item)
if os.path.exists(target_item_dir):
# Keep dangling links subject to the same replacement policy.
if os.path.lexists(target_item_dir):
if not upgrade:
logger.warning(
"Target directory %s already exists. Specify "
@@ -545,6 +822,7 @@ class InstallCommand(RequirementCommand):
target_item_dir,
)
continue
if os.path.islink(target_item_dir):
logger.warning(
"Target directory %s already exists and is "
@@ -554,6 +832,7 @@ class InstallCommand(RequirementCommand):
target_item_dir,
)
continue
if os.path.isdir(target_item_dir):
shutil.rmtree(target_item_dir)
else:
@@ -1,8 +1,9 @@
from __future__ import annotations
import contextlib
import json
import logging
from collections.abc import Generator, Sequence
from collections.abc import Generator, Iterator, Sequence
from email.parser import Parser
from optparse import Values
from typing import TYPE_CHECKING, cast
@@ -15,8 +16,8 @@ from pip._internal.cli.index_command import IndexGroupCommand
from pip._internal.cli.status_codes import SUCCESS
from pip._internal.exceptions import CommandError
from pip._internal.metadata import BaseDistribution, get_environment
from pip._internal.metadata.base import stdlib_pkgs
from pip._internal.models.selection_prefs import SelectionPreferences
from pip._internal.utils.compat import stdlib_pkgs
from pip._internal.utils.misc import tabulate, write_output
if TYPE_CHECKING:
@@ -135,9 +136,13 @@ class ListCommand(IndexGroupCommand):
self.parser.insert_option_group(0, selection_opts)
self.parser.insert_option_group(0, self.cmd_opts)
def handle_pip_version_check(self, options: Values) -> None:
if options.outdated or options.uptodate:
super().handle_pip_version_check(options)
@contextlib.contextmanager
def pip_version_check(self, options: Values, args: list[str]) -> Iterator[None]:
if not (options.outdated or options.uptodate):
yield
return
with super().pip_version_check(options, args):
yield
def _build_package_finder(
self, options: Values, session: PipSession
@@ -155,11 +160,14 @@ class ListCommand(IndexGroupCommand):
selection_prefs = SelectionPreferences(
allow_yanked=False,
release_control=options.release_control,
format_control=options.format_control,
prefer_binary=options.prefer_binary,
)
return PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
uploaded_prior_to=options.uploaded_prior_to,
)
def run(self, options: Values, args: list[str]) -> int:
@@ -175,10 +183,6 @@ class ListCommand(IndexGroupCommand):
cmdoptions.check_list_path_option(options)
skip = set(stdlib_pkgs)
if options.excludes:
skip.update(canonicalize_name(n) for n in options.excludes)
packages: _ProcessedDists = [
cast("_DistWithLatestInfo", d)
for d in get_environment(options.path).iter_installed_distributions(
@@ -186,7 +190,7 @@ class ListCommand(IndexGroupCommand):
user_only=options.user,
editables_only=options.editable,
include_editables=options.include_editable,
skip=skip,
skip=set(stdlib_pkgs),
)
]
@@ -197,6 +201,12 @@ class ListCommand(IndexGroupCommand):
if options.not_required:
packages = self.get_not_required(packages, options)
if options.excludes:
excluded_names = {canonicalize_name(n) for n in options.excludes}
packages = [
dist for dist in packages if dist.canonical_name not in excluded_names
]
if options.outdated:
packages = self.get_outdated(packages, options)
elif options.uptodate:
@@ -62,6 +62,7 @@ class LockCommand(RequirementCommand):
self.cmd_opts.add_option(cmdoptions.constraints())
self.cmd_opts.add_option(cmdoptions.build_constraints())
self.cmd_opts.add_option(cmdoptions.no_deps())
self.cmd_opts.add_option(cmdoptions.only_deps())
self.cmd_opts.add_option(cmdoptions.editable())
@@ -75,6 +76,7 @@ class LockCommand(RequirementCommand):
self.cmd_opts.add_option(cmdoptions.config_settings())
self.cmd_opts.add_option(cmdoptions.require_hashes())
self.cmd_opts.add_option(cmdoptions.no_require_hashes())
self.cmd_opts.add_option(cmdoptions.progress_bar())
index_opts = cmdoptions.make_option_group(
@@ -101,8 +103,9 @@ class LockCommand(RequirementCommand):
"without prior warning."
)
cmdoptions.check_build_constraints(options)
self.check_build_constraints(options)
cmdoptions.check_release_control_exclusive(options)
cmdoptions.check_only_deps_option_does_not_conflict(options)
session = self.get_default_session(options)
@@ -123,12 +126,6 @@ class LockCommand(RequirementCommand):
wheel_cache = WheelCache(options.cache_dir)
# Only when installing is it permitted to use PEP 660.
# In other circumstances (pip wheel, pip download) we generate
# regular (i.e. non editable) metadata and wheels.
for req in reqs:
req.permit_editable_wheels = True
preparer = self.make_requirement_preparer(
temp_build_dir=directory,
options=options,
@@ -137,6 +134,7 @@ class LockCommand(RequirementCommand):
finder=finder,
use_user_site=False,
verbosity=self.verbosity,
allow_editables=True,
)
resolver = self.make_resolver(
preparer=preparer,
@@ -116,18 +116,14 @@ def transform_hits(hits: list[dict[str, str]]) -> list[TransformedHit]:
def print_dist_installation_info(latest: str, dist: BaseDistribution | None) -> None:
if dist is not None:
with indent_log():
if dist.version == latest:
write_output("INSTALLED: %s (latest)", dist.version)
write_output("INSTALLED: %s", dist.version)
if parse_version(latest).pre:
write_output(
"LATEST: %s (pre-release; install with `pip install --pre`)",
latest,
)
else:
write_output("INSTALLED: %s", dist.version)
if parse_version(latest).pre:
write_output(
"LATEST: %s (pre-release; install"
" with `pip install --pre`)",
latest,
)
else:
write_output("LATEST: %s", latest)
write_output("LATEST: %s", latest)
def get_installed_distribution(name: str) -> BaseDistribution | None:
@@ -53,7 +53,7 @@ class ShowCommand(Command):
return ERROR
query = args
results = search_packages_info(query)
results = search_packages_info(query, include_files=options.files)
if not print_results(
results, list_files=options.files, verbose=options.verbose
):
@@ -82,7 +82,10 @@ class _PackageInfo(NamedTuple):
files: list[str] | None
def search_packages_info(query: list[str]) -> Generator[_PackageInfo, None, None]:
def search_packages_info(
query: list[str],
include_files: bool,
) -> Generator[_PackageInfo, None, None]:
"""
Gather details from installed distributions. Print distribution name,
version, location, and installed files. Installed files requires a
@@ -133,11 +136,11 @@ def search_packages_info(query: list[str]) -> Generator[_PackageInfo, None, None
except FileNotFoundError:
entry_points = []
files_iter = dist.iter_declared_entries()
if files_iter is None:
files: list[str] | None = None
else:
files = sorted(files_iter)
files: list[str] | None = None
if include_files:
files_iter = dist.iter_declared_entries()
if files_iter is not None:
files = sorted(files_iter)
metadata = dist.metadata
@@ -189,7 +192,10 @@ def print_results(
if i > 0:
write_output("---")
metadata_version_tuple = tuple(map(int, dist.metadata_version.split(".")))
metadata_version = dist.metadata_version
metadata_version_tuple = (
tuple(map(int, metadata_version.split("."))) if metadata_version else ()
)
write_output("Name: %s", dist.name)
write_output("Version: %s", dist.version)
@@ -62,6 +62,7 @@ class WheelCommand(RequirementCommand):
self.cmd_opts.add_option(cmdoptions.src())
self.cmd_opts.add_option(cmdoptions.ignore_requires_python())
self.cmd_opts.add_option(cmdoptions.no_deps())
self.cmd_opts.add_option(cmdoptions.only_deps())
self.cmd_opts.add_option(cmdoptions.progress_bar())
self.cmd_opts.add_option(
@@ -75,6 +76,7 @@ class WheelCommand(RequirementCommand):
self.cmd_opts.add_option(cmdoptions.config_settings())
self.cmd_opts.add_option(cmdoptions.require_hashes())
self.cmd_opts.add_option(cmdoptions.no_require_hashes())
index_opts = cmdoptions.make_option_group(
cmdoptions.index_group,
@@ -92,8 +94,9 @@ class WheelCommand(RequirementCommand):
@with_cleanup
def run(self, options: Values, args: list[str]) -> int:
cmdoptions.check_build_constraints(options)
self.check_build_constraints(options)
cmdoptions.check_release_control_exclusive(options)
cmdoptions.check_only_deps_option_does_not_conflict(options)
session = self.get_default_session(options)
@@ -123,6 +126,7 @@ class WheelCommand(RequirementCommand):
download_dir=options.wheel_dir,
use_user_site=False,
verbosity=self.verbosity,
allow_editables=False,
)
resolver = self.make_resolver(
@@ -151,6 +155,7 @@ class WheelCommand(RequirementCommand):
reqs_to_build,
wheel_cache=wheel_cache,
verify=(not options.no_verify),
allow_editables=False,
)
for req in build_successes:
assert req.link and req.link.is_wheel
@@ -160,8 +165,9 @@ class WheelCommand(RequirementCommand):
shutil.copy(req.local_file_path, options.wheel_dir)
except OSError as e:
logger.warning(
"Building wheel for %s failed: %s",
"Failed to copy built wheel %s to %s: %s",
req.name,
options.wheel_dir,
e,
)
build_failures.append(req)
@@ -14,37 +14,44 @@ Some terminology:
from __future__ import annotations
import configparser
import locale
import os
import sys
from collections.abc import Iterable
from typing import Any, NewType
from enum import Enum
from typing import Any
from pip._internal.exceptions import (
ConfigurationError,
ConfigurationFileCouldNotBeLoaded,
)
from pip._internal.utils import appdirs
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.logging import getLogger
from pip._internal.utils.misc import ensure_dir, enum
from pip._internal.utils._log import getLogger
from pip._internal.utils.compat import WINDOWS, get_locale_encoding
from pip._internal.utils.misc import ensure_dir
RawConfigParser = configparser.RawConfigParser # Shorthand
Kind = NewType("Kind", str)
CONFIG_BASENAME = "pip.ini" if WINDOWS else "pip.conf"
ENV_NAMES_IGNORED = "version", "help"
# The kinds of configurations there are.
kinds = enum(
USER="user", # User Specific
GLOBAL="global", # System Wide
SITE="site", # [Virtual] Environment Specific
ENV="env", # from PIP_CONFIG_FILE
ENV_VAR="env-var", # from Environment Variables
)
OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE
# TODO: Use StrEnum when Python 3.10 support is dropped.
class Kind(str, Enum):
"""Configuration sources, including files and environment variables."""
USER = "user" # User Specific
GLOBAL = "global" # System Wide
SITE = "site" # [Virtual] Environment Specific
ENV = "env" # from PIP_CONFIG_FILE
ENV_VAR = "env-var" # from Environment Variables
__str__ = str.__str__
__repr__ = str.__repr__
OVERRIDE_ORDER = Kind.GLOBAL, Kind.USER, Kind.SITE, Kind.ENV, Kind.ENV_VAR
VALID_LOAD_ONLY = Kind.USER, Kind.GLOBAL, Kind.SITE
logger = getLogger(__name__)
@@ -80,12 +87,19 @@ def get_configuration_files() -> dict[Kind, list[str]]:
)
new_config_file = os.path.join(appdirs.user_config_dir("pip"), CONFIG_BASENAME)
return {
kinds.GLOBAL: global_config_files,
kinds.SITE: [site_config_file],
kinds.USER: [legacy_config_file, new_config_file],
Kind.GLOBAL: global_config_files,
Kind.SITE: [site_config_file],
Kind.USER: [legacy_config_file, new_config_file],
}
class _RawConfigParser(configparser.RawConfigParser):
"""RawConfigParser with normalized key names."""
def optionxform(self, optionstr: str) -> str:
return _normalize_name(optionstr)
class Configuration:
"""Handles management of configuration.
@@ -120,6 +134,7 @@ class Configuration:
variant: {} for variant in OVERRIDE_ORDER
}
self._modified_parsers: list[tuple[str, RawConfigParser]] = []
self._use_env = False
def load(self) -> None:
"""Loads configuration from configuration files and environment"""
@@ -133,7 +148,7 @@ class Configuration:
try:
return self._get_parser_to_modify()[0]
except IndexError:
except ConfigurationError:
return None
def items(self) -> Iterable[tuple[str, Any]]:
@@ -184,12 +199,10 @@ class Configuration:
self._ensure_have_load_only()
assert self.load_only
fname, parser = self._get_parser_to_modify()
if (
key not in self._config[self.load_only][fname]
and key not in self._config[self.load_only]
):
fname, parser = self._get_parser_to_modify(key)
load_config = Kind.ENV if self._use_env else self.load_only
file_config = self._config[load_config].get(fname, {})
if key not in file_config:
raise ConfigurationError(f"No such key - {orig_key}")
if parser is not None:
@@ -207,9 +220,9 @@ class Configuration:
parser.remove_section(section)
self._mark_as_modified(fname, parser)
try:
del self._config[self.load_only][fname][key]
del self._config[load_config][fname][key]
except KeyError:
del self._config[self.load_only][key]
del self._config[load_config][key]
def save(self) -> None:
"""Save the current in-memory state."""
@@ -255,20 +268,27 @@ class Configuration:
def _load_config_files(self) -> None:
"""Loads configuration from configuration files"""
config_files = dict(self.iter_config_files())
if config_files[kinds.ENV][0:1] == [os.devnull]:
if config_files[Kind.ENV][0:1] == [os.devnull]:
logger.debug(
"Skipping loading configuration files due to "
"environment's PIP_CONFIG_FILE being os.devnull"
)
return
env_config_file = os.environ.get("PIP_CONFIG_FILE")
for variant, files in config_files.items():
for fname in files:
# If there's specific variant set in `load_only`, load only
# that variant, not the others.
if self.load_only is not None and variant != self.load_only:
logger.debug("Skipping file '%s' (variant: %s)", fname, variant)
continue
is_env_redirect = (
variant == Kind.ENV
and fname == env_config_file
and self.load_only == Kind.USER
)
if not is_env_redirect:
logger.debug("Skipping file '%s' (variant: %s)", fname, variant)
continue
parser = self._load_file(variant, fname)
@@ -279,27 +299,27 @@ class Configuration:
logger.verbose("For variant '%s', will try loading '%s'", variant, fname)
parser = self._construct_parser(fname)
self._config[variant].setdefault(fname, {})
for section in parser.sections():
items = parser.items(section)
self._config[variant].setdefault(fname, {})
self._config[variant][fname].update(self._normalized_keys(section, items))
return parser
def _construct_parser(self, fname: str) -> RawConfigParser:
parser = configparser.RawConfigParser()
parser = _RawConfigParser()
# If there is no such file, don't bother reading it but create the
# parser anyway, to hold the data.
# Doing this is useful when modifying and saving files, where we don't
# need to construct a parser.
if os.path.exists(fname):
locale_encoding = locale.getpreferredencoding(False)
encoding = "utf-8" if sys.flags.utf8_mode else get_locale_encoding()
try:
parser.read(fname, encoding=locale_encoding)
parser.read(fname, encoding=encoding)
except UnicodeDecodeError:
# See https://github.com/pypa/pip/issues/4963
raise ConfigurationFileCouldNotBeLoaded(
reason=f"contains invalid {locale_encoding} characters",
reason=f"contains invalid {encoding} characters",
fname=fname,
)
except configparser.Error as error:
@@ -309,8 +329,8 @@ class Configuration:
def _load_environment_vars(self) -> None:
"""Loads configuration from environment variables"""
self._config[kinds.ENV_VAR].setdefault(":env:", {})
self._config[kinds.ENV_VAR][":env:"].update(
self._config[Kind.ENV_VAR].setdefault(":env:", {})
self._config[Kind.ENV_VAR][":env:"].update(
self._normalized_keys(":env:", self.get_environ_vars())
)
@@ -351,7 +371,7 @@ class Configuration:
env_config_file = os.environ.get("PIP_CONFIG_FILE", None)
config_files = get_configuration_files()
yield kinds.GLOBAL, config_files[kinds.GLOBAL]
yield Kind.GLOBAL, config_files[Kind.GLOBAL]
# per-user config is not loaded when env_config_file exists
should_load_user_config = not self.isolated and not (
@@ -359,30 +379,58 @@ class Configuration:
)
if should_load_user_config:
# The legacy config file is overridden by the new config file
yield kinds.USER, config_files[kinds.USER]
yield Kind.USER, config_files[Kind.USER]
# virtualenv config
yield kinds.SITE, config_files[kinds.SITE]
yield Kind.SITE, config_files[Kind.SITE]
if env_config_file is not None:
yield kinds.ENV, [env_config_file]
yield Kind.ENV, [env_config_file]
else:
yield kinds.ENV, []
yield Kind.ENV, []
def get_values_in_config(self, variant: Kind) -> dict[str, Any]:
"""Get values present in a config file"""
return self._config[variant]
def _get_parser_to_modify(self) -> tuple[str, RawConfigParser]:
def _get_parser_to_modify(
self, key: str | None = None
) -> tuple[str, RawConfigParser]:
# Determine which parser to modify
assert self.load_only
parsers = self._parsers[self.load_only]
if not parsers:
# This should not happen if everything works correctly.
env_config_file = os.environ.get("PIP_CONFIG_FILE")
if env_config_file == os.devnull:
raise ConfigurationError(
"Cannot write to PIP_CONFIG_VALUE when it is set to "
f"{env_config_file}"
)
if env_config_file and self.load_only in (Kind.USER, Kind.GLOBAL):
parser = self._construct_parser(env_config_file)
self._parsers[self.load_only].append((env_config_file, parser))
self._use_env = True
logger.warning(
"Because PIP_CONFIG_FILE is set, changes will be applied to "
"'%s', not to '%s'. To restore normal behavior, "
"unset PIP_CONFIG_FILE.",
env_config_file,
self.load_only,
)
return env_config_file, parser
raise ConfigurationError(
"Fatal Internal error [id=2]. Please report as a bug."
)
if key is None:
return parsers[-1]
section, name = _disassemble_key(key)
for fname, parser in reversed(parsers):
if parser.has_section(section) and parser.has_option(section, name):
return fname, parser
# Use the highest priority parser.
return parsers[-1]
@@ -7,7 +7,7 @@ from pip._internal.metadata.base import BaseDistribution
from pip._internal.req import InstallRequirement
if TYPE_CHECKING:
from pip._internal.build_env import BuildEnvironmentInstaller
from pip._internal.build_env import BuildEnvironmentInstaller, BuildIsolationMode
class AbstractDistribution(metaclass=abc.ABCMeta):
@@ -33,7 +33,8 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
super().__init__()
self.req = req
@abc.abstractproperty
@property
@abc.abstractmethod
def build_tracker_id(self) -> str | None:
"""A string that uniquely identifies this requirement to the build tracker.
@@ -49,7 +50,8 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
def prepare_distribution_metadata(
self,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
build_isolation: BuildIsolationMode,
check_build_deps: bool,
allow_editables: bool,
) -> None:
raise NotImplementedError()
@@ -6,7 +6,7 @@ from pip._internal.distributions.base import AbstractDistribution
from pip._internal.metadata import BaseDistribution
if TYPE_CHECKING:
from pip._internal.build_env import BuildEnvironmentInstaller
from pip._internal.build_env import BuildEnvironmentInstaller, BuildIsolationMode
class InstalledDistribution(AbstractDistribution):
@@ -27,7 +27,8 @@ class InstalledDistribution(AbstractDistribution):
def prepare_distribution_metadata(
self,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
build_isolation: BuildIsolationMode,
check_build_deps: bool,
allow_editables: bool,
) -> None:
pass
@@ -2,9 +2,15 @@ from __future__ import annotations
import logging
from collections.abc import Iterable
from contextlib import nullcontext
from typing import TYPE_CHECKING
from pip._internal.build_env import BuildEnvironment
from pip._internal.build_env import (
BuildIsolationMode,
NoOpBuildEnvironment,
VenvBuildEnvironment,
VirtualBuildEnvironment,
)
from pip._internal.distributions.base import AbstractDistribution
from pip._internal.exceptions import InstallationError
from pip._internal.metadata import BaseDistribution
@@ -35,24 +41,28 @@ class SourceDistribution(AbstractDistribution):
def prepare_distribution_metadata(
self,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
build_isolation: BuildIsolationMode,
check_build_deps: bool,
allow_editables: bool,
) -> None:
# Load pyproject.toml
# Load pyproject.toml and set up backend environment
self.req.load_pyproject_toml()
self._prepare_build_env(build_isolation, build_env_installer)
# Set up the build isolation, if this requirement should be isolated
if build_isolation:
if build_isolation != "off":
# Setup an isolated environment and install the build backend static
# requirements in it.
self._prepare_build_backend(build_env_installer)
self._prepare_build_backend(build_isolation)
# Check that the build backend supports PEP 660. This cannot be done
# earlier because we need to setup the build backend to verify it
# supports build_editable, nor can it be done later, because we want
# to avoid installing build requirements needlessly.
self.req.editable_sanity_check()
# Install the dynamic build requirements.
self._install_build_reqs(build_env_installer)
self._install_build_reqs(
build_env_installer, build_isolation, allow_editables=allow_editables
)
else:
# When not using build isolation, we still need to check that
# the build backend supports PEP 660.
@@ -68,20 +78,34 @@ class SourceDistribution(AbstractDistribution):
self._raise_conflicts("the backend dependencies", conflicting)
if missing:
self._raise_missing_reqs(missing)
self.req.prepare_metadata()
self.req.prepare_metadata(allow_editables)
def _prepare_build_backend(
self, build_env_installer: BuildEnvironmentInstaller
def _prepare_build_env(
self,
build_isolation: BuildIsolationMode,
build_env_installer: BuildEnvironmentInstaller,
) -> None:
# Isolate in a BuildEnvironment and install the build-time
# requirements.
if build_isolation == "virtual":
self.req.build_env = VirtualBuildEnvironment(build_env_installer)
elif build_isolation == "venv":
self.req.build_env = VenvBuildEnvironment(build_env_installer)
self.req.configure_backend(self.req.build_env.python_executable)
def _prepare_build_backend(self, build_isolation: BuildIsolationMode) -> None:
# Install the pyproject.toml declared build-time requirements.
pyproject_requires = self.req.pyproject_requires
assert pyproject_requires is not None
assert not isinstance(self.req.build_env, NoOpBuildEnvironment)
self.req.build_env = BuildEnvironment(build_env_installer)
self.req.build_env.install_requirements(
pyproject_requires, "overlay", kind="build dependencies", for_req=self.req
)
context = self.req.build_env if build_isolation == "venv" else nullcontext()
with context:
self.req.build_env.install_requirements(
pyproject_requires,
"overlay",
kind="build dependencies",
for_req=self.req,
)
conflicting, missing = self.req.build_env.check_requirements(
self.req.requirements_to_check
)
@@ -117,14 +141,17 @@ class SourceDistribution(AbstractDistribution):
return backend.get_requires_for_build_editable()
def _install_build_reqs(
self, build_env_installer: BuildEnvironmentInstaller
self,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: BuildIsolationMode,
allow_editables: bool,
) -> None:
# Install any extra build dependencies that the backend requests.
# This must be done in a second pass, as the pyproject.toml
# dependencies must be installed before we can call the backend.
if (
self.req.editable
and self.req.permit_editable_wheels
and allow_editables
and self.req.supports_pyproject_editable
):
build_reqs = self._get_build_requires_editable()
@@ -133,9 +160,11 @@ class SourceDistribution(AbstractDistribution):
conflicting, missing = self.req.build_env.check_requirements(build_reqs)
if conflicting:
self._raise_conflicts("the backend dependencies", conflicting)
self.req.build_env.install_requirements(
missing, "normal", kind="backend dependencies", for_req=self.req
)
context = self.req.build_env if build_isolation == "venv" else nullcontext()
with context:
self.req.build_env.install_requirements(
missing, "normal", kind="backend dependencies", for_req=self.req
)
def _raise_conflicts(
self, conflicting_with: str, conflicting_reqs: set[tuple[str, str]]
@@ -12,7 +12,7 @@ from pip._internal.metadata import (
)
if TYPE_CHECKING:
from pip._internal.build_env import BuildEnvironmentInstaller
from pip._internal.build_env import BuildEnvironmentInstaller, BuildIsolationMode
class WheelDistribution(AbstractDistribution):
@@ -38,7 +38,8 @@ class WheelDistribution(AbstractDistribution):
def prepare_distribution_metadata(
self,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
build_isolation: BuildIsolationMode,
check_build_deps: bool,
allow_editables: bool,
) -> None:
pass
@@ -1,971 +0,0 @@
"""Exceptions used throughout package.
This module MUST NOT try to import from anything within `pip._internal` to
operate. This is expected to be importable from any/all files within the
subpackage and, thus, should not depend on them.
"""
from __future__ import annotations
import configparser
import contextlib
import locale
import logging
import pathlib
import re
import sys
import traceback
from collections.abc import Iterable, Iterator
from itertools import chain, groupby, repeat
from typing import TYPE_CHECKING, Literal
from pip._vendor.packaging.requirements import InvalidRequirement
from pip._vendor.packaging.version import InvalidVersion
from pip._vendor.rich.console import Console, ConsoleOptions, RenderResult
from pip._vendor.rich.markup import escape
from pip._vendor.rich.text import Text
if TYPE_CHECKING:
from hashlib import _Hash
from pip._vendor.requests.models import PreparedRequest, Request, Response
from pip._internal.metadata import BaseDistribution
from pip._internal.models.link import Link
from pip._internal.network.download import _FileDownload
from pip._internal.req.req_install import InstallRequirement
logger = logging.getLogger(__name__)
#
# Scaffolding
#
def _is_kebab_case(s: str) -> bool:
return re.match(r"^[a-z]+(-[a-z]+)*$", s) is not None
def _prefix_with_indent(
s: Text | str,
console: Console,
*,
prefix: str,
indent: str,
) -> Text:
if isinstance(s, Text):
text = s
else:
text = console.render_str(s)
return console.render_str(prefix, overflow="ignore") + console.render_str(
f"\n{indent}", overflow="ignore"
).join(text.split(allow_blank=True))
class PipError(Exception):
"""The base pip error."""
class DiagnosticPipError(PipError):
"""An error, that presents diagnostic information to the user.
This contains a bunch of logic, to enable pretty presentation of our error
messages. Each error gets a unique reference. Each error can also include
additional context, a hint and/or a note -- which are presented with the
main error message in a consistent style.
This is adapted from the error output styling in `sphinx-theme-builder`.
"""
reference: str
def __init__(
self,
*,
kind: Literal["error", "warning"] = "error",
reference: str | None = None,
message: str | Text,
context: str | Text | None,
hint_stmt: str | Text | None,
note_stmt: str | Text | None = None,
link: str | None = None,
) -> None:
# Ensure a proper reference is provided.
if reference is None:
assert hasattr(self, "reference"), "error reference not provided!"
reference = self.reference
assert _is_kebab_case(reference), "error reference must be kebab-case!"
self.kind = kind
self.reference = reference
self.message = message
self.context = context
self.note_stmt = note_stmt
self.hint_stmt = hint_stmt
self.link = link
super().__init__(f"<{self.__class__.__name__}: {self.reference}>")
def __repr__(self) -> str:
return (
f"<{self.__class__.__name__}("
f"reference={self.reference!r}, "
f"message={self.message!r}, "
f"context={self.context!r}, "
f"note_stmt={self.note_stmt!r}, "
f"hint_stmt={self.hint_stmt!r}"
")>"
)
def __rich_console__(
self,
console: Console,
options: ConsoleOptions,
) -> RenderResult:
colour = "red" if self.kind == "error" else "yellow"
yield f"[{colour} bold]{self.kind}[/]: [bold]{self.reference}[/]"
yield ""
if not options.ascii_only:
# Present the main message, with relevant context indented.
if self.context is not None:
yield _prefix_with_indent(
self.message,
console,
prefix=f"[{colour}]×[/] ",
indent=f"[{colour}]│[/] ",
)
yield _prefix_with_indent(
self.context,
console,
prefix=f"[{colour}]╰─>[/] ",
indent=f"[{colour}] [/] ",
)
else:
yield _prefix_with_indent(
self.message,
console,
prefix="[red]×[/] ",
indent=" ",
)
else:
yield self.message
if self.context is not None:
yield ""
yield self.context
if self.note_stmt is not None or self.hint_stmt is not None:
yield ""
if self.note_stmt is not None:
yield _prefix_with_indent(
self.note_stmt,
console,
prefix="[magenta bold]note[/]: ",
indent=" ",
)
if self.hint_stmt is not None:
yield _prefix_with_indent(
self.hint_stmt,
console,
prefix="[cyan bold]hint[/]: ",
indent=" ",
)
if self.link is not None:
yield ""
yield f"Link: {self.link}"
#
# Actual Errors
#
class ConfigurationError(PipError):
"""General exception in configuration"""
class InstallationError(PipError):
"""General exception during installation"""
class FailedToPrepareCandidate(InstallationError):
"""Raised when we fail to prepare a candidate (i.e. fetch and generate metadata).
This is intentionally not a diagnostic error, since the output will be presented
above this error, when this occurs. This should instead present information to the
user.
"""
def __init__(
self, *, package_name: str, requirement_chain: str, failed_step: str
) -> None:
super().__init__(f"Failed to build '{package_name}' when {failed_step.lower()}")
self.package_name = package_name
self.requirement_chain = requirement_chain
self.failed_step = failed_step
class MissingPyProjectBuildRequires(DiagnosticPipError):
"""Raised when pyproject.toml has `build-system`, but no `build-system.requires`."""
reference = "missing-pyproject-build-system-requires"
def __init__(self, *, package: str) -> None:
super().__init__(
message=f"Can not process {escape(package)}",
context=Text(
"This package has an invalid pyproject.toml file.\n"
"The [build-system] table is missing the mandatory `requires` key."
),
note_stmt="This is an issue with the package mentioned above, not pip.",
hint_stmt=Text("See PEP 518 for the detailed specification."),
)
class InvalidPyProjectBuildRequires(DiagnosticPipError):
"""Raised when pyproject.toml an invalid `build-system.requires`."""
reference = "invalid-pyproject-build-system-requires"
def __init__(self, *, package: str, reason: str) -> None:
super().__init__(
message=f"Can not process {escape(package)}",
context=Text(
"This package has an invalid `build-system.requires` key in "
f"pyproject.toml.\n{reason}"
),
note_stmt="This is an issue with the package mentioned above, not pip.",
hint_stmt=Text("See PEP 518 for the detailed specification."),
)
class NoneMetadataError(PipError):
"""Raised when accessing a Distribution's "METADATA" or "PKG-INFO".
This signifies an inconsistency, when the Distribution claims to have
the metadata file (if not, raise ``FileNotFoundError`` instead), but is
not actually able to produce its content. This may be due to permission
errors.
"""
def __init__(
self,
dist: BaseDistribution,
metadata_name: str,
) -> None:
"""
:param dist: A Distribution object.
:param metadata_name: The name of the metadata being accessed
(can be "METADATA" or "PKG-INFO").
"""
self.dist = dist
self.metadata_name = metadata_name
def __str__(self) -> str:
# Use `dist` in the error message because its stringification
# includes more information, like the version and location.
return f"None {self.metadata_name} metadata found for distribution: {self.dist}"
class UserInstallationInvalid(InstallationError):
"""A --user install is requested on an environment without user site."""
def __str__(self) -> str:
return "User base directory is not specified"
class InvalidSchemeCombination(InstallationError):
def __str__(self) -> str:
before = ", ".join(str(a) for a in self.args[:-1])
return f"Cannot set {before} and {self.args[-1]} together"
class DistributionNotFound(InstallationError):
"""Raised when a distribution cannot be found to satisfy a requirement"""
class RequirementsFileParseError(InstallationError):
"""Raised when a general error occurs parsing a requirements file line."""
class BestVersionAlreadyInstalled(PipError):
"""Raised when the most up-to-date version of a package is already
installed."""
class BadCommand(PipError):
"""Raised when virtualenv or a command is not found"""
class CommandError(PipError):
"""Raised when there is an error in command-line arguments"""
class PreviousBuildDirError(PipError):
"""Raised when there's a previous conflicting build directory"""
class NetworkConnectionError(PipError):
"""HTTP connection error"""
def __init__(
self,
error_msg: str,
response: Response | None = None,
request: Request | PreparedRequest | None = None,
) -> None:
"""
Initialize NetworkConnectionError with `request` and `response`
objects.
"""
self.response = response
self.request = request
self.error_msg = error_msg
if (
self.response is not None
and not self.request
and hasattr(response, "request")
):
self.request = self.response.request
super().__init__(error_msg, response, request)
def __str__(self) -> str:
return str(self.error_msg)
class InvalidWheelFilename(InstallationError):
"""Invalid wheel filename."""
class UnsupportedWheel(InstallationError):
"""Unsupported wheel."""
class InvalidWheel(InstallationError):
"""Invalid (e.g. corrupt) wheel."""
def __init__(self, location: str, name: str):
self.location = location
self.name = name
def __str__(self) -> str:
return f"Wheel '{self.name}' located at {self.location} is invalid."
class MetadataInconsistent(InstallationError):
"""Built metadata contains inconsistent information.
This is raised when the metadata contains values (e.g. name and version)
that do not match the information previously obtained from sdist filename,
user-supplied ``#egg=`` value, or an install requirement name.
"""
def __init__(
self, ireq: InstallRequirement, field: str, f_val: str, m_val: str
) -> None:
self.ireq = ireq
self.field = field
self.f_val = f_val
self.m_val = m_val
def __str__(self) -> str:
return (
f"Requested {self.ireq} has inconsistent {self.field}: "
f"expected {self.f_val!r}, but metadata has {self.m_val!r}"
)
class MetadataInvalid(InstallationError):
"""Metadata is invalid."""
def __init__(self, ireq: InstallRequirement, error: str) -> None:
self.ireq = ireq
self.error = error
def __str__(self) -> str:
return f"Requested {self.ireq} has invalid metadata: {self.error}"
class InstallationSubprocessError(DiagnosticPipError, InstallationError):
"""A subprocess call failed."""
reference = "subprocess-exited-with-error"
def __init__(
self,
*,
command_description: str,
exit_code: int,
output_lines: list[str] | None,
) -> None:
if output_lines is None:
output_prompt = Text("No available output.")
else:
output_prompt = (
Text.from_markup(f"[red][{len(output_lines)} lines of output][/]\n")
+ Text("".join(output_lines))
+ Text.from_markup(R"[red]\[end of output][/]")
)
super().__init__(
message=(
f"[green]{escape(command_description)}[/] did not run successfully.\n"
f"exit code: {exit_code}"
),
context=output_prompt,
hint_stmt=None,
note_stmt=(
"This error originates from a subprocess, and is likely not a "
"problem with pip."
),
)
self.command_description = command_description
self.exit_code = exit_code
def __str__(self) -> str:
return f"{self.command_description} exited with {self.exit_code}"
class MetadataGenerationFailed(DiagnosticPipError, InstallationError):
reference = "metadata-generation-failed"
def __init__(
self,
*,
package_details: str,
) -> None:
super().__init__(
message="Encountered error while generating package metadata.",
context=escape(package_details),
hint_stmt="See above for details.",
note_stmt="This is an issue with the package mentioned above, not pip.",
)
def __str__(self) -> str:
return "metadata generation failed"
class HashErrors(InstallationError):
"""Multiple HashError instances rolled into one for reporting"""
def __init__(self) -> None:
self.errors: list[HashError] = []
def append(self, error: HashError) -> None:
self.errors.append(error)
def __str__(self) -> str:
lines = []
self.errors.sort(key=lambda e: e.order)
for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__):
lines.append(cls.head)
lines.extend(e.body() for e in errors_of_cls)
if lines:
return "\n".join(lines)
return ""
def __bool__(self) -> bool:
return bool(self.errors)
class HashError(InstallationError):
"""
A failure to verify a package against known-good hashes
:cvar order: An int sorting hash exception classes by difficulty of
recovery (lower being harder), so the user doesn't bother fretting
about unpinned packages when he has deeper issues, like VCS
dependencies, to deal with. Also keeps error reports in a
deterministic order.
:cvar head: A section heading for display above potentially many
exceptions of this kind
:ivar req: The InstallRequirement that triggered this error. This is
pasted on after the exception is instantiated, because it's not
typically available earlier.
"""
req: InstallRequirement | None = None
head = ""
order: int = -1
def body(self) -> str:
"""Return a summary of me for display under the heading.
This default implementation simply prints a description of the
triggering requirement.
:param req: The InstallRequirement that provoked this error, with
its link already populated by the resolver's _populate_link().
"""
return f" {self._requirement_name()}"
def __str__(self) -> str:
return f"{self.head}\n{self.body()}"
def _requirement_name(self) -> str:
"""Return a description of the requirement that triggered me.
This default implementation returns long description of the req, with
line numbers
"""
return str(self.req) if self.req else "unknown package"
class VcsHashUnsupported(HashError):
"""A hash was provided for a version-control-system-based requirement, but
we don't have a method for hashing those."""
order = 0
head = (
"Can't verify hashes for these requirements because we don't "
"have a way to hash version control repositories:"
)
class DirectoryUrlHashUnsupported(HashError):
"""A hash was provided for a version-control-system-based requirement, but
we don't have a method for hashing those."""
order = 1
head = (
"Can't verify hashes for these file:// requirements because they "
"point to directories:"
)
class HashMissing(HashError):
"""A hash was needed for a requirement but is absent."""
order = 2
head = (
"Hashes are required in --require-hashes mode, but they are "
"missing from some requirements. Here is a list of those "
"requirements along with the hashes their downloaded archives "
"actually had. Add lines like these to your requirements files to "
"prevent tampering. (If you did not enable --require-hashes "
"manually, note that it turns on automatically when any package "
"has a hash.)"
)
def __init__(self, gotten_hash: str) -> None:
"""
:param gotten_hash: The hash of the (possibly malicious) archive we
just downloaded
"""
self.gotten_hash = gotten_hash
def body(self) -> str:
# Dodge circular import.
from pip._internal.utils.hashes import FAVORITE_HASH
package = None
if self.req:
# In the case of URL-based requirements, display the original URL
# seen in the requirements file rather than the package name,
# so the output can be directly copied into the requirements file.
package = (
self.req.original_link
if self.req.is_direct
# In case someone feeds something downright stupid
# to InstallRequirement's constructor.
else getattr(self.req, "req", None)
)
return " {} --hash={}:{}".format(
package or "unknown package", FAVORITE_HASH, self.gotten_hash
)
class HashUnpinned(HashError):
"""A requirement had a hash specified but was not pinned to a specific
version."""
order = 3
head = (
"In --require-hashes mode, all requirements must have their "
"versions pinned with ==. These do not:"
)
class HashMismatch(HashError):
"""
Distribution file hash values don't match.
:ivar package_name: The name of the package that triggered the hash
mismatch. Feel free to write to this after the exception is raise to
improve its error message.
"""
order = 4
head = (
"THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS "
"FILE. If you have updated the package versions, please update "
"the hashes. Otherwise, examine the package contents carefully; "
"someone may have tampered with them."
)
def __init__(self, allowed: dict[str, list[str]], gots: dict[str, _Hash]) -> None:
"""
:param allowed: A dict of algorithm names pointing to lists of allowed
hex digests
:param gots: A dict of algorithm names pointing to hashes we
actually got from the files under suspicion
"""
self.allowed = allowed
self.gots = gots
def body(self) -> str:
return f" {self._requirement_name()}:\n{self._hash_comparison()}"
def _hash_comparison(self) -> str:
"""
Return a comparison of actual and expected hash values.
Example::
Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde
or 123451234512345123451234512345123451234512345
Got bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef
"""
def hash_then_or(hash_name: str) -> chain[str]:
# For now, all the decent hashes have 6-char names, so we can get
# away with hard-coding space literals.
return chain([hash_name], repeat(" or"))
lines: list[str] = []
for hash_name, expecteds in self.allowed.items():
prefix = hash_then_or(hash_name)
lines.extend((f" Expected {next(prefix)} {e}") for e in expecteds)
lines.append(
f" Got {self.gots[hash_name].hexdigest()}\n"
)
return "\n".join(lines)
class UnsupportedPythonVersion(InstallationError):
"""Unsupported python version according to Requires-Python package
metadata."""
class ConfigurationFileCouldNotBeLoaded(ConfigurationError):
"""When there are errors while loading a configuration file"""
def __init__(
self,
reason: str = "could not be loaded",
fname: str | None = None,
error: configparser.Error | None = None,
) -> None:
super().__init__(error)
self.reason = reason
self.fname = fname
self.error = error
def __str__(self) -> str:
if self.fname is not None:
message_part = f" in {self.fname}."
else:
assert self.error is not None
message_part = f".\n{self.error}\n"
return f"Configuration file {self.reason}{message_part}"
_DEFAULT_EXTERNALLY_MANAGED_ERROR = f"""\
The Python environment under {sys.prefix} is managed externally, and may not be
manipulated by the user. Please use specific tooling from the distributor of
the Python installation to interact with this environment instead.
"""
class ExternallyManagedEnvironment(DiagnosticPipError):
"""The current environment is externally managed.
This is raised when the current environment is externally managed, as
defined by `PEP 668`_. The ``EXTERNALLY-MANAGED`` configuration is checked
and displayed when the error is bubbled up to the user.
:param error: The error message read from ``EXTERNALLY-MANAGED``.
"""
reference = "externally-managed-environment"
def __init__(self, error: str | None) -> None:
if error is None:
context = Text(_DEFAULT_EXTERNALLY_MANAGED_ERROR)
else:
context = Text(error)
super().__init__(
message="This environment is externally managed",
context=context,
note_stmt=(
"If you believe this is a mistake, please contact your "
"Python installation or OS distribution provider. "
"You can override this, at the risk of breaking your Python "
"installation or OS, by passing --break-system-packages."
),
hint_stmt=Text("See PEP 668 for the detailed specification."),
)
@staticmethod
def _iter_externally_managed_error_keys() -> Iterator[str]:
# LC_MESSAGES is in POSIX, but not the C standard. The most common
# platform that does not implement this category is Windows, where
# using other categories for console message localization is equally
# unreliable, so we fall back to the locale-less vendor message. This
# can always be re-evaluated when a vendor proposes a new alternative.
try:
category = locale.LC_MESSAGES
except AttributeError:
lang: str | None = None
else:
lang, _ = locale.getlocale(category)
if lang is not None:
yield f"Error-{lang}"
for sep in ("-", "_"):
before, found, _ = lang.partition(sep)
if not found:
continue
yield f"Error-{before}"
yield "Error"
@classmethod
def from_config(
cls,
config: pathlib.Path | str,
) -> ExternallyManagedEnvironment:
parser = configparser.ConfigParser(interpolation=None)
try:
parser.read(config, encoding="utf-8")
section = parser["externally-managed"]
for key in cls._iter_externally_managed_error_keys():
with contextlib.suppress(KeyError):
return cls(section[key])
except KeyError:
pass
except (OSError, UnicodeDecodeError, configparser.ParsingError):
from pip._internal.utils._log import VERBOSE
exc_info = logger.isEnabledFor(VERBOSE)
logger.warning("Failed to read %s", config, exc_info=exc_info)
return cls(None)
class UninstallMissingRecord(DiagnosticPipError):
reference = "uninstall-no-record-file"
def __init__(self, *, distribution: BaseDistribution) -> None:
installer = distribution.installer
if not installer or installer == "pip":
dep = f"{distribution.raw_name}=={distribution.version}"
hint = Text.assemble(
"You might be able to recover from this via: ",
(f"pip install --force-reinstall --no-deps {dep}", "green"),
)
else:
hint = Text(
f"The package was installed by {installer}. "
"You should check if it can uninstall the package."
)
super().__init__(
message=Text(f"Cannot uninstall {distribution}"),
context=(
"The package's contents are unknown: "
f"no RECORD file was found for {distribution.raw_name}."
),
hint_stmt=hint,
)
class LegacyDistutilsInstall(DiagnosticPipError):
reference = "uninstall-distutils-installed-package"
def __init__(self, *, distribution: BaseDistribution) -> None:
super().__init__(
message=Text(f"Cannot uninstall {distribution}"),
context=(
"It is a distutils installed project and thus we cannot accurately "
"determine which files belong to it which would lead to only a partial "
"uninstall."
),
hint_stmt=None,
)
class InvalidInstalledPackage(DiagnosticPipError):
reference = "invalid-installed-package"
def __init__(
self,
*,
dist: BaseDistribution,
invalid_exc: InvalidRequirement | InvalidVersion,
) -> None:
installed_location = dist.installed_location
if isinstance(invalid_exc, InvalidRequirement):
invalid_type = "requirement"
else:
invalid_type = "version"
super().__init__(
message=Text(
f"Cannot process installed package {dist} "
+ (f"in {installed_location!r} " if installed_location else "")
+ f"because it has an invalid {invalid_type}:\n{invalid_exc.args[0]}"
),
context=(
"Starting with pip 24.1, packages with invalid "
f"{invalid_type}s can not be processed."
),
hint_stmt="To proceed this package must be uninstalled.",
)
class IncompleteDownloadError(DiagnosticPipError):
"""Raised when the downloader receives fewer bytes than advertised
in the Content-Length header."""
reference = "incomplete-download"
def __init__(self, download: _FileDownload) -> None:
# Dodge circular import.
from pip._internal.utils.misc import format_size
assert download.size is not None
download_status = (
f"{format_size(download.bytes_received)}/{format_size(download.size)}"
)
if download.reattempts:
retry_status = f"after {download.reattempts + 1} attempts "
hint = "Use --resume-retries to configure resume attempt limit."
else:
# Download retrying is not enabled.
retry_status = ""
hint = "Consider using --resume-retries to enable download resumption."
message = Text(
f"Download failed {retry_status}because not enough bytes "
f"were received ({download_status})"
)
super().__init__(
message=message,
context=f"URL: {download.link.redacted_url}",
hint_stmt=hint,
note_stmt="This is an issue with network connectivity, not pip.",
)
class ResolutionTooDeepError(DiagnosticPipError):
"""Raised when the dependency resolver exceeds the maximum recursion depth."""
reference = "resolution-too-deep"
def __init__(self) -> None:
super().__init__(
message="Dependency resolution exceeded maximum depth",
context=(
"Pip cannot resolve the current dependencies as the dependency graph "
"is too complex for pip to solve efficiently."
),
hint_stmt=(
"Try adding lower bounds to constrain your dependencies, "
"for example: 'package>=2.0.0' instead of just 'package'. "
),
link="https://pip.pypa.io/en/stable/topics/dependency-resolution/#handling-resolution-too-deep-errors",
)
class InstallWheelBuildError(DiagnosticPipError):
reference = "failed-wheel-build-for-install"
def __init__(self, failed: list[InstallRequirement]) -> None:
super().__init__(
message=(
"Failed to build installable wheels for some "
"pyproject.toml based projects"
),
context=", ".join(r.name for r in failed), # type: ignore
hint_stmt=None,
)
class InvalidEggFragment(DiagnosticPipError):
reference = "invalid-egg-fragment"
def __init__(self, link: Link, fragment: str) -> None:
hint = ""
if ">" in fragment or "=" in fragment or "<" in fragment:
hint = (
"Version specifiers are silently ignored for URL references. "
"Remove them. "
)
if "[" in fragment and "]" in fragment:
hint += "Try using the Direct URL requirement syntax: 'name[extra] @ URL'"
if not hint:
hint = "Egg fragments can only be a valid project name."
super().__init__(
message=f"The '{escape(fragment)}' egg fragment is invalid",
context=f"from '{escape(str(link))}'",
hint_stmt=escape(hint),
)
class BuildDependencyInstallError(DiagnosticPipError):
"""Raised when build dependencies cannot be installed."""
reference = "failed-build-dependency-install"
def __init__(
self,
req: InstallRequirement | None,
build_reqs: Iterable[str],
*,
cause: Exception,
log_lines: list[str] | None,
) -> None:
if isinstance(cause, PipError):
note = "This is likely not a problem with pip."
else:
note = (
"pip crashed unexpectedly. Please file an issue on pip's issue "
"tracker: https://github.com/pypa/pip/issues/new"
)
if log_lines is None:
# No logs are available, they must have been printed earlier.
context = Text("See above for more details.")
else:
if isinstance(cause, PipError):
log_lines.append(f"ERROR: {cause}")
else:
# Split rendered error into real lines without trailing newlines.
log_lines.extend(
"".join(traceback.format_exception(cause)).splitlines()
)
context = Text.assemble(
f"Installing {' '.join(build_reqs)}\n",
(f"[{len(log_lines)} lines of output]\n", "red"),
"\n".join(log_lines),
("\n[end of output]", "red"),
)
message = Text("Cannot install build dependencies", "green")
if req:
message += Text(f" for {req}")
super().__init__(
message=message, context=context, hint_stmt=None, note_stmt=note
)
@@ -12,21 +12,27 @@ import json
import logging
import os
import urllib.parse
from collections.abc import Iterable, MutableMapping, Sequence
from collections.abc import Callable, Iterable, MutableMapping, Sequence
from dataclasses import dataclass
from html.parser import HTMLParser
from optparse import Values
from typing import (
Callable,
NamedTuple,
Protocol,
)
from pip._vendor import requests
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.requests import Response
from pip._vendor.requests.exceptions import RetryError, SSLError
from pip._vendor.requests.exceptions import RetryError
from pip._internal.exceptions import NetworkConnectionError
from pip._internal.exceptions import (
ConnectionFailedError,
ConnectionTimeoutError,
NetworkConnectionError,
ProxyConnectionError,
SSLMissingError,
SSLVerificationError,
)
from pip._internal.models.link import Link
from pip._internal.models.search_scope import SearchScope
from pip._internal.network.session import PipSession
@@ -80,7 +86,7 @@ def _ensure_api_header(response: Response) -> None:
):
return
raise _NotAPIContent(content_type, response.request.method)
raise _NotAPIContent(content_type, response.request.method) # type: ignore[arg-type]
class _NotHTTP(Exception):
@@ -105,7 +111,11 @@ def _ensure_api_response(url: str, session: PipSession) -> None:
_ensure_api_header(resp)
def _get_simple_response(url: str, session: PipSession) -> Response:
def _get_simple_response(
url: str,
session: PipSession,
force_revalidate: bool = False,
) -> Response:
"""Access an Simple API response with GET, and return the response.
This consists of three parts:
@@ -123,32 +133,24 @@ def _get_simple_response(url: str, session: PipSession) -> Response:
logger.debug("Getting page %s", redact_auth_from_url(url))
resp = session.get(
url,
headers={
"Accept": ", ".join(
[
"application/vnd.pypi.simple.v1+json",
"application/vnd.pypi.simple.v1+html; q=0.1",
"text/html; q=0.01",
]
),
# We don't want to blindly returned cached data for
# /simple/, because authors generally expecting that
# twine upload && pip install will function, but if
# they've done a pip install in the last ~10 minutes
# it won't. Thus by setting this to zero we will not
# blindly use any cached data, however the benefit of
# using max-age=0 instead of no-cache, is that we will
# still support conditional requests, so we will still
# minimize traffic sent in cases where the page hasn't
# changed at all, we will just always incur the round
# trip for the conditional GET now instead of only
# once per 10 minutes.
# For more information, please see pypa/pip#5670.
"Cache-Control": "max-age=0",
},
)
headers = {
"Accept": ", ".join(
[
"application/vnd.pypi.simple.v1+json",
"application/vnd.pypi.simple.v1+html; q=0.1",
"text/html; q=0.01",
]
),
}
if force_revalidate:
# Using max-age=0 rather than no-cache still supports conditional
# requests, minimizing traffic when the page hasn't changed.
# See pypa/pip#5670.
logger.debug("Refreshing package index response.")
headers["Cache-Control"] = "max-age=0"
resp = session.get(url, headers=headers)
raise_for_status(resp)
# The check for archives above only works if the url ends with
@@ -316,7 +318,12 @@ def _make_index_content(
)
def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | None:
def _get_index_content(
link: Link,
*,
session: PipSession,
force_revalidate: bool = False,
) -> IndexContent | None:
url = link.url.split("#", 1)[0]
# Check for VCS schemes that do not support lookup as web pages.
@@ -343,7 +350,9 @@ def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | Non
logger.debug(" file: URL is directory, getting %s", url)
try:
resp = _get_simple_response(url, session=session)
resp = _get_simple_response(
url, session=session, force_revalidate=force_revalidate
)
except _NotHTTP:
logger.warning(
"Skipping page %s because it looks like an archive, and cannot "
@@ -359,18 +368,17 @@ def _get_index_content(link: Link, *, session: PipSession) -> IndexContent | Non
exc.request_desc,
exc.content_type,
)
except NetworkConnectionError as exc:
except (RetryError, NetworkConnectionError) as exc:
_handle_get_simple_fail(link, exc)
except RetryError as exc:
_handle_get_simple_fail(link, exc)
except SSLError as exc:
reason = "There was a problem confirming the ssl certificate: "
reason += str(exc)
except (SSLVerificationError, SSLMissingError) as exc:
reason = f"There was a problem confirming the ssl certificate: {exc.context}"
_handle_get_simple_fail(link, reason, meth=logger.info)
except requests.ConnectionError as exc:
_handle_get_simple_fail(link, f"connection error: {exc}")
except requests.Timeout:
_handle_get_simple_fail(link, "timed out")
except ConnectionFailedError as exc:
_handle_get_simple_fail(link, f"connection error: {exc.context}")
except ProxyConnectionError as exc:
_handle_get_simple_fail(link, f"proxy connection error: {exc.context}")
except ConnectionTimeoutError as exc:
_handle_get_simple_fail(link, str(exc.context))
else:
return _make_index_content(resp, cache_link_parsing=link.cache_link_parsing)
return None
@@ -435,11 +443,25 @@ class LinkCollector:
def find_links(self) -> list[str]:
return self.search_scope.find_links
def fetch_response(self, location: Link) -> IndexContent | None:
def fetch_response(
self, location: Link, package_name: str | None = None
) -> IndexContent | None:
"""
Fetch an HTML page containing package links.
"""
return _get_index_content(location, session=self.session)
# By default we may return cached /simple/ responses. Users can force
# revalidation to ensure newly uploaded distributions become visible
# immediately after publication. See pypa/pip#5670.
force_revalidate = self.session.refresh_package
should_force_revalidate = ":all:" in force_revalidate or (
package_name is not None
and canonicalize_name(package_name) in force_revalidate
)
return _get_index_content(
location,
session=self.session,
force_revalidate=should_force_revalidate,
)
def collect_sources(
self,
@@ -12,14 +12,12 @@ from collections.abc import Iterable
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Optional,
Union,
)
from pip._vendor.packaging import specifiers
from pip._vendor.packaging.tags import Tag
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import InvalidVersion, Version, _BaseVersion
from pip._vendor.packaging.version import InvalidVersion, _BaseVersion
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import (
@@ -49,15 +47,15 @@ from pip._internal.utils.packaging import check_requires_python
from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
if TYPE_CHECKING:
from typing_extensions import TypeGuard
from typing import TypeGuard
__all__ = ["FormatControl", "BestCandidateResult", "PackageFinder"]
logger = getLogger(__name__)
BuildTag = Union[tuple[()], tuple[int, str]]
CandidateSortingKey = tuple[int, int, int, _BaseVersion, Optional[int], BuildTag]
BuildTag = tuple[()] | tuple[int, str]
CandidateSortingKey = tuple[int, int, int, _BaseVersion, int | None, BuildTag]
def _check_link_requires_python(
@@ -137,7 +135,7 @@ class LinkEvaluator:
formats: frozenset[str],
target_python: TargetPython,
allow_yanked: bool,
ignore_requires_python: bool | None = None,
ignore_requires_python: bool = False,
uploaded_prior_to: datetime.datetime | None = None,
) -> None:
"""
@@ -159,8 +157,6 @@ class LinkEvaluator:
:param uploaded_prior_to: If set, only allow links uploaded prior to
the given datetime.
"""
if ignore_requires_python is None:
ignore_requires_python = False
self._allow_yanked = allow_yanked
self._canonical_name = canonical_name
@@ -493,7 +489,6 @@ class CandidateEvaluator:
else:
allow_prereleases = None
specifier = self._specifier
# When using the pkg_resources backend we turn the version object into
# a str here because otherwise when we're debundled but setuptools isn't,
# Python will see packaging.version.Version and
@@ -501,22 +496,17 @@ class CandidateEvaluator:
# types. This way we'll use a str as a common data interchange
# format. If we stop using the pkg_resources provided specifier
# and start using our own, we can drop the cast to str().
if select_backend().NAME == "pkg_resources":
candidates_and_versions: list[
tuple[InstallationCandidate, str | Version]
] = [(c, str(c.version)) for c in candidates]
else:
candidates_and_versions = [(c, c.version) for c in candidates]
versions = set(
specifier.filter(
(v for _, v in candidates_and_versions),
prereleases=allow_prereleases,
)
applicable_candidates = specifier.filter(
candidates,
prereleases=allow_prereleases,
key=lambda c: (
str(c.version)
if select_backend().NAME == "pkg_resources"
else c.version
),
)
applicable_candidates = [c for c, v in candidates_and_versions if v in versions]
filtered_applicable_candidates = filter_unallowed_hashes(
candidates=applicable_candidates,
candidates=list(applicable_candidates),
hashes=self._hashes,
project_name=self._project_name,
)
@@ -633,7 +623,7 @@ class PackageFinder:
allow_yanked: bool,
format_control: FormatControl | None = None,
candidate_prefs: CandidatePreferences | None = None,
ignore_requires_python: bool | None = None,
ignore_requires_python: bool = False,
uploaded_prior_to: datetime.datetime | None = None,
) -> None:
"""
@@ -660,8 +650,10 @@ class PackageFinder:
self.format_control = format_control
# These are boring links that have already been logged somehow.
self._logged_links: set[tuple[Link, LinkType, str]] = set()
# Collects the detail strings for links skipped due to Requires-Python
# incompatibility. Used by requires_python_skipped_reasons() to build
# the error message when resolution fails.
self._requires_python_skipped: set[str] = set()
# Cache of the result of finding candidates
self._all_candidates: dict[str, list[InstallationCandidate]] = {}
@@ -670,6 +662,9 @@ class PackageFinder:
BestCandidateResult,
] = {}
# projects for which a link is locked from a pylock
self._locked_links: dict[NormalizedName, Link] = {}
# Don't include an allow_yanked default value to make sure each call
# site considers whether yanked releases are allowed. This also causes
# that decision to be made explicit in the calling code, which helps
@@ -730,10 +725,18 @@ class PackageFinder:
def index_urls(self) -> list[str]:
return self.search_scope.index_urls
@property
def refresh_package(self) -> set[str]:
return self._link_collector.session.refresh_package
@property
def proxy(self) -> str | None:
return self._link_collector.session.pip_proxy
@property
def no_proxy_env(self) -> bool:
return self._link_collector.session.pip_no_proxy_env
@property
def trusted_hosts(self) -> Iterable[str]:
for host_port in self._link_collector.session.pip_trusted_origins:
@@ -772,12 +775,7 @@ class PackageFinder:
return self._uploaded_prior_to
def requires_python_skipped_reasons(self) -> list[str]:
reasons = {
detail
for _, result, detail in self._logged_links
if result == LinkType.requires_python_mismatch
}
return sorted(reasons)
return sorted(self._requires_python_skipped)
def make_link_evaluator(self, project_name: str) -> LinkEvaluator:
canonical_name = canonicalize_name(project_name)
@@ -810,12 +808,11 @@ class PackageFinder:
return no_eggs + eggs
def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None:
entry = (link, result, detail)
if entry not in self._logged_links:
# Put the link at the end so the reason is more visible and because
# the link string is usually very long.
logger.debug("Skipping link: %s: %s", detail, link)
self._logged_links.add(entry)
# Put the link at the end so the reason is more visible and because
# the link string is usually very long.
logger.debug("Skipping link: %s: %s", detail, link)
if result == LinkType.requires_python_mismatch:
self._requires_python_skipped.add(detail)
def get_install_candidate(
self, link_evaluator: LinkEvaluator, link: Link
@@ -863,7 +860,9 @@ class PackageFinder:
"Fetching project page and analyzing links: %s",
project_url,
)
index_response = self._link_collector.fetch_response(project_url)
index_response = self._link_collector.fetch_response(
project_url, package_name=link_evaluator.project_name
)
if index_response is None:
return []
@@ -880,7 +879,8 @@ class PackageFinder:
def find_all_candidates(self, project_name: str) -> list[InstallationCandidate]:
"""Find all available InstallationCandidate for project_name
This checks index_urls and find_links.
This checks index_urls and find_links, unless a locked link is known
for that project.
All versions found are returned as an InstallationCandidate list.
See LinkEvaluator.evaluate_link() for details on which files
@@ -891,6 +891,23 @@ class PackageFinder:
link_evaluator = self.make_link_evaluator(project_name)
if locked_link := self._locked_links.get(canonicalize_name(project_name)):
# If a locked link is known for that project, do not check
# index_urls nor find_links. We don't use get_install_candidate here,
# because if a locked link is unsupported (due to format control,
# release control or otherwise), we want to error out immediately
# instead of ignoring it.
result, detail = link_evaluator.evaluate_link(locked_link)
if result != LinkType.candidate:
raise InstallationError(
f"Could not install locked package {project_name!r} "
f"from {locked_link.comes_from!r}: {detail}"
)
self._all_candidates[project_name] = [
InstallationCandidate(project_name, detail, locked_link, locked=True)
]
return self._all_candidates[project_name]
collected_sources = self._link_collector.collect_sources(
project_name=project_name,
candidates_from_page=functools.partial(
@@ -1028,16 +1045,16 @@ class PackageFinder:
)
if allows_pre is False:
version_type = "final version"
req_disp = req.format_without_marker()
logger.critical(
"Could not find a %s that satisfies the requirement %s "
"(from versions: %s)",
version_type,
req,
req_disp,
_format_versions(best_candidate_result.all_candidates),
)
raise DistributionNotFound(f"No matching distribution found for {req}")
raise DistributionNotFound(f"No matching distribution found for {req_disp}")
def _should_install_candidate(
candidate: InstallationCandidate | None,
@@ -1080,6 +1097,16 @@ class PackageFinder:
)
raise BestVersionAlreadyInstalled
def add_locked_link(self, project_name: NormalizedName, locked_link: Link) -> None:
assert not self._all_candidates
if project_name in self._locked_links:
raise InstallationError(
f"Multiple locked links provided for {project_name}: "
f"{self._locked_links[project_name]} and {locked_link}"
)
self._locked_links[project_name] = locked_link
def _find_name_version_sep(fragment: str, canonical_name: str) -> int:
"""Find the separator's index based on the package's canonical name.
@@ -4,8 +4,7 @@ import logging
import mimetypes
import os
from collections import defaultdict
from collections.abc import Iterable
from typing import Callable
from collections.abc import Callable, Iterable
from pip._vendor.packaging.utils import (
InvalidSdistFilename,
@@ -41,13 +41,11 @@ logger = logging.getLogger(__name__)
_PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
_USE_SYSCONFIG_DEFAULT = sys.version_info >= (3, 10)
def _should_use_sysconfig() -> bool:
"""This function determines the value of _USE_SYSCONFIG.
By default, pip uses sysconfig on Python 3.10+.
By default, pip uses sysconfig.
But Python distributors can override this decision by setting:
sysconfig._PIP_USE_SYSCONFIG = True / False
Rationale in https://github.com/pypa/pip/issues/10647
@@ -55,7 +53,7 @@ def _should_use_sysconfig() -> bool:
This is a function for testability, but should be constant during any one
run.
"""
return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", _USE_SYSCONFIG_DEFAULT))
return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", True))
_USE_SYSCONFIG = _should_use_sysconfig()
@@ -68,10 +66,10 @@ if not _USE_SYSCONFIG:
# Be noisy about incompatibilities if this platforms "should" be using
# sysconfig, but is explicitly opting out and using distutils instead.
if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG:
_MISMATCH_LEVEL = logging.WARNING
else:
if _USE_SYSCONFIG:
_MISMATCH_LEVEL = logging.DEBUG
else:
_MISMATCH_LEVEL = logging.WARNING
def _looks_like_bpo_44860() -> bool:
@@ -281,7 +279,7 @@ def get_scheme(
if k == "platlib" and _looks_like_red_hat_lib():
continue
# On Python 3.9+, sysconfig's posix_user scheme sets platlib against
# sysconfig's posix_user scheme sets platlib against
# sys.platlibdir, but distutils's unix_user incorrectly continues
# using the same $usersite for both platlib and purelib. This creates a
# mismatch when sys.platlibdir is not "lib".
@@ -20,11 +20,10 @@ import logging
import os
import sys
from distutils.cmd import Command as DistutilsCommand
from distutils.command.install import SCHEME_KEYS
from distutils.command.install import install as distutils_install_command
from distutils.sysconfig import get_python_lib
from pip._internal.models.scheme import Scheme
from pip._internal.models.scheme import SCHEME_KEYS, Scheme
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -4,7 +4,7 @@ import logging
import os
import sys
import sysconfig
from typing import Callable
from collections.abc import Callable
from pip._internal.exceptions import InvalidSchemeCombination, UserInstallationInvalid
from pip._internal.models.scheme import SCHEME_KEYS, Scheme
@@ -158,6 +158,14 @@ def get_scheme(
else:
scheme_name = _infer_prefix()
# Special case: When installing into a custom prefix, use posix_prefix
# instead of posix_local because it should *only* be used to redirecting
# to /usr/local for system packager installs. Notably, posix_local is used
# on Debian and its derivatives.
# See also: https://discuss.python.org/t/18240
if prefix is not None and scheme_name == "posix_local":
scheme_name = "posix_prefix"
# Special case: When installing into a custom prefix, use posix_prefix
# instead of osx_framework_library. See _should_use_osx_framework_prefix()
# docstring for details.
@@ -41,7 +41,7 @@ def change_root(new_root: str, pathname: str) -> str:
return os.path.join(new_root, pathname[1:])
elif os.name == "nt":
(drive, path) = os.path.splitdrive(pathname)
drive, path = os.path.splitdrive(pathname)
if path[0] == "\\":
path = path[1:]
return os.path.join(new_root, path)
@@ -80,7 +80,7 @@ def _emit_pkg_resources_deprecation_if_needed() -> None:
"to use the default importlib.metadata backend, "
"by unsetting the _PIP_USE_IMPORTLIB_METADATA environment variable"
),
gone_in="26.3",
gone_in="27.0",
issue=13317,
)
@@ -14,7 +14,6 @@ from typing import (
Any,
NamedTuple,
Protocol,
Union,
)
from pip._vendor.packaging.requirements import Requirement
@@ -29,14 +28,21 @@ from pip._internal.models.direct_url import (
DirectUrl,
DirectUrlValidationError,
)
from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
from pip._internal.utils.egg_link import egg_link_path_from_sys_path
from pip._internal.utils.misc import is_local, normalize_path
from pip._internal.utils.urls import url_to_path
from ._json import msg_to_json
InfoPath = Union[str, pathlib.PurePath]
# packages in the stdlib that may have installation metadata, but should not be
# considered 'installed'. this theoretically could be determined based on
# dist.location (py27:`sysconfig.get_paths()['stdlib']`,
# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may
# make this ineffective, so hard-coding
stdlib_pkgs = {"python", "wsgiref", "argparse"}
InfoPath = str | pathlib.PurePath
logger = logging.getLogger(__name__)
@@ -152,14 +158,13 @@ class BaseDistribution(Protocol):
"""
raise NotImplementedError()
@property
@functools.cached_property
def editable_project_location(self) -> str | None:
"""The project location for editable distributions.
This is the directory where pyproject.toml or setup.py is located.
None if the distribution is not installed in editable mode.
"""
# TODO: this property is relatively costly to compute, memoize it ?
direct_url = self.direct_url
if direct_url:
if direct_url.is_local_editable():
@@ -617,11 +622,29 @@ class BaseEnvironment:
flags=re.IGNORECASE,
)
if not project_name_valid:
logger.warning(
"Ignoring invalid distribution %s (%s)",
dist.canonical_name,
dist.location,
# Check the directory name rather than the distribution name,
# since the pkg_resources (default below 3.11) backend normalizes
# the leading tilde to a dash.
# TODO: use dist.canonical_name for this check once
# pkg_resources support is dropped (#13317).
info_location = dist.info_location
leftover_name = (
pathlib.Path(info_location).name if info_location else ""
)
if leftover_name.startswith("~"):
logger.warning(
"Ignoring incompletely removed distribution %s (%s); "
"'~'-prefixed leftover directories are the result of a "
"previous incomplete installation, and can be deleted",
leftover_name,
dist.location,
)
else:
logger.warning(
"Ignoring invalid distribution %s (%s)",
dist.canonical_name,
dist.location,
)
continue
yield dist
@@ -15,7 +15,7 @@ from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import InvalidWheel, UnsupportedWheel
from pip._internal.exceptions import InstallationError, InvalidWheel, UnsupportedWheel
from pip._internal.metadata.base import (
BaseDistribution,
BaseEntryPoint,
@@ -177,7 +177,7 @@ class Distribution(BaseDistribution):
@property
def raw_version(self) -> str:
return self._dist.version
return self.metadata["Version"]
def is_file(self, path: InfoPath) -> bool:
return self._dist.read_text(str(path)) is not None
@@ -199,7 +199,13 @@ class Distribution(BaseDistribution):
def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
# importlib.metadata's EntryPoint structure satisfies BaseEntryPoint.
return self._dist.entry_points
try:
return self._dist.entry_points
except ValueError as e:
# Python 3.15+ validates entry points while parsing.
raise InstallationError(
f"Error parsing entry points for {self.raw_name}: {e}"
) from e
def _metadata_impl(self) -> email.message.Message:
# From Python 3.10+, importlib.metadata declares PackageMetadata as the
@@ -207,7 +213,13 @@ class Distribution(BaseDistribution):
# a ton of fields that we need, including get() and get_payload(). We
# rely on the implementation that the object is actually a Message now,
# until upstream can improve the protocol. (python/cpython#94952)
return cast(email.message.Message, self._dist.metadata)
metadata = self._dist.metadata
# From Python 3.15+, importlib.metadata may return None when no
# metadata file (METADATA or PKG-INFO) exists in the distribution
# directory. (python/cpython#132947)
if metadata is None:
return email.message.Message()
return cast(email.message.Message, metadata)
def iter_provided_extras(self) -> Iterable[NormalizedName]:
return [
@@ -7,7 +7,6 @@ import pathlib
import sys
import zipfile
from collections.abc import Iterator, Sequence
from typing import Optional
from pip._vendor.packaging.utils import (
InvalidWheelFilename,
@@ -24,6 +23,10 @@ from ._dists import Distribution
logger = logging.getLogger(__name__)
# Used to avoid emitting duplicate invalid distribution metadata warnings for
# the same dist-info directory.
_warned_bad_metadata: set[BasePath | None] = set()
def _looks_like_wheel(location: str) -> bool:
if not location.endswith(WHEEL_EXTENSION):
@@ -50,7 +53,7 @@ class _DistributionFinder:
installations as well. It's useful feature, after all.
"""
FoundResult = tuple[importlib.metadata.Distribution, Optional[BasePath]]
FoundResult = tuple[importlib.metadata.Distribution, BasePath | None]
def __init__(self) -> None:
self._found_names: set[NormalizedName] = set()
@@ -69,7 +72,9 @@ class _DistributionFinder:
try:
name = get_dist_canonical_name(dist)
except BadMetadata as e:
logger.warning("Skipping %s due to %s", info_location, e.reason)
if info_location not in _warned_bad_metadata:
logger.warning("Skipping %s due to %s", info_location, e.reason)
_warned_bad_metadata.add(info_location)
continue
if name in self._found_names:
continue
@@ -16,7 +16,12 @@ from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import InvalidWheel, NoneMetadataError, UnsupportedWheel
from pip._internal.exceptions import (
InstallationError,
InvalidWheel,
NoneMetadataError,
UnsupportedWheel,
)
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.misc import display_path, normalize_path
from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
@@ -208,7 +213,13 @@ class Distribution(BaseDistribution):
return content
def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
for group, entries in self._dist.get_entry_map().items():
try:
entry_map = self._dist.get_entry_map()
except ValueError as e:
raise InstallationError(
f"Error parsing entry points for {self.raw_name}: {e}"
) from e
for group, entries in entry_map.items():
for name, entry_point in entries.items():
name, _, value = str(entry_point).partition("=")
yield EntryPoint(name=name.strip(), value=value.strip(), group=group)
@@ -6,20 +6,22 @@ from pip._vendor.packaging.version import parse as parse_version
from pip._internal.models.link import Link
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class InstallationCandidate:
"""Represents a potential "candidate" for installation."""
__slots__ = ["name", "version", "link"]
name: str
version: Version
link: Link
locked: bool
def __init__(self, name: str, version: str, link: Link) -> None:
def __init__(
self, name: str, version: str, link: Link, locked: bool = False
) -> None:
object.__setattr__(self, "name", name)
object.__setattr__(self, "version", parse_version(version))
object.__setattr__(self, "link", link)
object.__setattr__(self, "locked", locked)
def __str__(self) -> str:
return f"{self.name!r} candidate (version {self.version} at {self.link})"
@@ -3,225 +3,40 @@
from __future__ import annotations
import json
import re
import urllib.parse
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, ClassVar, TypeVar, Union
from typing import Any
from pip._vendor.packaging.direct_url import (
ArchiveInfo,
DirectUrlValidationError,
DirInfo,
VcsInfo,
)
from pip._vendor.packaging.direct_url import (
DirectUrl as PackagingDirectUrl,
)
__all__ = [
"ArchiveInfo",
"DirInfo",
"DirectUrl",
"DirectUrlValidationError",
"DirInfo",
"ArchiveInfo",
"DIRECT_URL_METADATA_NAME",
"VcsInfo",
]
T = TypeVar("T")
DIRECT_URL_METADATA_NAME = "direct_url.json"
ENV_VAR_RE = re.compile(r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$")
class DirectUrlValidationError(Exception):
pass
def _get(
d: dict[str, Any], expected_type: type[T], key: str, default: T | None = None
) -> T | None:
"""Get value from dictionary and verify expected type."""
if key not in d:
return default
value = d[key]
if not isinstance(value, expected_type):
raise DirectUrlValidationError(
f"{value!r} has unexpected type for {key} (expected {expected_type})"
)
return value
def _get_required(
d: dict[str, Any], expected_type: type[T], key: str, default: T | None = None
) -> T:
value = _get(d, expected_type, key, default)
if value is None:
raise DirectUrlValidationError(f"{key} must have a value")
return value
def _exactly_one_of(infos: Iterable[InfoType | None]) -> InfoType:
infos = [info for info in infos if info is not None]
if not infos:
raise DirectUrlValidationError(
"missing one of archive_info, dir_info, vcs_info"
)
if len(infos) > 1:
raise DirectUrlValidationError(
"more than one of archive_info, dir_info, vcs_info"
)
assert infos[0] is not None
return infos[0]
def _filter_none(**kwargs: Any) -> dict[str, Any]:
"""Make dict excluding None values."""
return {k: v for k, v in kwargs.items() if v is not None}
@dataclass
class VcsInfo:
name: ClassVar = "vcs_info"
vcs: str
commit_id: str
requested_revision: str | None = None
@classmethod
def _from_dict(cls, d: dict[str, Any] | None) -> VcsInfo | None:
if d is None:
return None
return cls(
vcs=_get_required(d, str, "vcs"),
commit_id=_get_required(d, str, "commit_id"),
requested_revision=_get(d, str, "requested_revision"),
)
def _to_dict(self) -> dict[str, Any]:
return _filter_none(
vcs=self.vcs,
requested_revision=self.requested_revision,
commit_id=self.commit_id,
)
class ArchiveInfo:
name = "archive_info"
def __init__(
self,
hash: str | None = None,
hashes: dict[str, str] | None = None,
) -> None:
# set hashes before hash, since the hash setter will further populate hashes
self.hashes = hashes
self.hash = hash
@property
def hash(self) -> str | None:
return self._hash
@hash.setter
def hash(self, value: str | None) -> None:
if value is not None:
# Auto-populate the hashes key to upgrade to the new format automatically.
# We don't back-populate the legacy hash key from hashes.
try:
hash_name, hash_value = value.split("=", 1)
except ValueError:
raise DirectUrlValidationError(
f"invalid archive_info.hash format: {value!r}"
)
if self.hashes is None:
self.hashes = {hash_name: hash_value}
elif hash_name not in self.hashes:
self.hashes = self.hashes.copy()
self.hashes[hash_name] = hash_value
self._hash = value
@classmethod
def _from_dict(cls, d: dict[str, Any] | None) -> ArchiveInfo | None:
if d is None:
return None
return cls(hash=_get(d, str, "hash"), hashes=_get(d, dict, "hashes"))
def _to_dict(self) -> dict[str, Any]:
return _filter_none(hash=self.hash, hashes=self.hashes)
@dataclass
class DirInfo:
name: ClassVar = "dir_info"
editable: bool = False
@classmethod
def _from_dict(cls, d: dict[str, Any] | None) -> DirInfo | None:
if d is None:
return None
return cls(editable=_get_required(d, bool, "editable", default=False))
def _to_dict(self) -> dict[str, Any]:
return _filter_none(editable=self.editable or None)
InfoType = Union[ArchiveInfo, DirInfo, VcsInfo]
@dataclass
class DirectUrl:
url: str
info: InfoType
subdirectory: str | None = None
def _remove_auth_from_netloc(self, netloc: str) -> str:
if "@" not in netloc:
return netloc
user_pass, netloc_no_user_pass = netloc.split("@", 1)
if (
isinstance(self.info, VcsInfo)
and self.info.vcs == "git"
and user_pass == "git"
):
return netloc
if ENV_VAR_RE.match(user_pass):
return netloc
return netloc_no_user_pass
@property
def redacted_url(self) -> str:
"""url with user:password part removed unless it is formed with
environment variables as specified in PEP 610, or it is ``git``
in the case of a git URL.
"""
purl = urllib.parse.urlsplit(self.url)
netloc = self._remove_auth_from_netloc(purl.netloc)
surl = urllib.parse.urlunsplit(
(purl.scheme, netloc, purl.path, purl.query, purl.fragment)
)
return surl
def validate(self) -> None:
self.from_dict(self.to_dict())
@classmethod
def from_dict(cls, d: dict[str, Any]) -> DirectUrl:
return DirectUrl(
url=_get_required(d, str, "url"),
subdirectory=_get(d, str, "subdirectory"),
info=_exactly_one_of(
[
ArchiveInfo._from_dict(_get(d, dict, "archive_info")),
DirInfo._from_dict(_get(d, dict, "dir_info")),
VcsInfo._from_dict(_get(d, dict, "vcs_info")),
]
),
)
def to_dict(self) -> dict[str, Any]:
res = _filter_none(
url=self.redacted_url,
subdirectory=self.subdirectory,
)
res[self.info.name] = self.info._to_dict()
return res
class DirectUrl(PackagingDirectUrl):
def to_dict_compat(self) -> dict[str, Any]:
return dict(super().to_dict(generate_legacy_hash=True))
@classmethod
def from_json(cls, s: str) -> DirectUrl:
return cls.from_dict(json.loads(s))
def to_json(self) -> str:
return json.dumps(self.to_dict(), sort_keys=True)
return json.dumps(self.to_dict_compat(), sort_keys=True)
def is_local_editable(self) -> bool:
return isinstance(self.info, DirInfo) and self.info.editable
return bool(self.dir_info and self.dir_info.editable)
@@ -19,7 +19,7 @@ class InstallationReport:
# be absent when the requirement was installed from the wheel cache
# and the cache entry was populated by an older pip version that did not
# record origin.json.
"download_info": ireq.download_info.to_dict(),
"download_info": ireq.download_info.to_dict_compat(),
# is_direct is true if the requirement was a direct URL reference (which
# includes editable requirements), and false if the requirement was
# downloaded from a PEP 503 index or --find-links.
@@ -8,12 +8,12 @@ import os
import posixpath
import re
import urllib.parse
import urllib.request
from collections.abc import Mapping
from dataclasses import dataclass
from typing import (
Any,
NamedTuple,
NewType,
)
from pip._internal.exceptions import InvalidEggFragment
@@ -31,6 +31,49 @@ from pip._internal.utils.urls import path_to_url, url_to_path
logger = logging.getLogger(__name__)
# A single path component: percent-decoded once and reduced to a basename, so it
# contains no path separator and is not a ``.`` or ``..`` reference. The empty
# string means "no component".
PathComponent = NewType("PathComponent", str)
def _to_path_component(name: str) -> PathComponent:
"""Reduce ``name`` to a single path component, or ``""`` if it has none.
``os.path.basename`` drops any directory part, drive letter, or separator;
a ``.``, ``..``, or empty result is not a component and becomes ``""``.
"""
name = os.path.basename(name)
if name in ("", os.curdir, os.pardir):
return PathComponent("")
return PathComponent(name)
def as_path_component(name: str) -> PathComponent:
"""Like ``_to_path_component`` but reject the empty result.
Use where a file is about to be written, so a missing name is an error
rather than a silent fallback to the directory itself.
"""
component = _to_path_component(name)
if not component:
raise ValueError(f"Unexpected file name derived from URL: {name!r}")
return component
def join_within_directory(directory: str, component: PathComponent) -> str:
"""Join a single path ``component`` onto ``directory``.
``component`` is a :data:`PathComponent`, so by type it has no separator and
is not a ``.`` or ``..`` reference; the result can never escape ``directory``.
Requiring ``PathComponent`` rather than ``str`` lets the type checker enforce
at the call site that the name was reduced to a safe component beforehand.
"""
return os.path.join(directory, component)
# Order matters, earlier hashes have a precedence over later hashes for what
# we will pick to use.
_SUPPORTED_HASHES = ("sha512", "sha384", "sha256", "sha224", "sha1", "md5")
@@ -125,6 +168,8 @@ def _clean_file_url_path(part: str) -> str:
Clean the first part of a URL path that corresponds to a local
filesystem path (i.e. the first part after splitting on "@" characters).
"""
import urllib.request
# We unquote prior to quoting to make sure nothing is double quoted.
# Also, on Windows the path part might contain a drive letter which
# should not be quoted. On Linux where drive letters do not
@@ -423,18 +468,13 @@ class Link:
return redact_auth_from_url(self.url)
@property
def filename(self) -> str:
path = self.path.rstrip("/")
name = posixpath.basename(path)
if not name:
# Make sure we don't leak auth information if the netloc
# includes a username and password.
netloc, user_pass = split_auth_from_netloc(self.netloc)
return netloc
def filename(self) -> PathComponent:
name = _to_path_component(posixpath.basename(self.path.rstrip("/")))
if name:
return name
name = urllib.parse.unquote(name)
assert name, f"URL {self._url!r} produced no filename"
return name
# No component in the path; fall back to the netloc, dropping any auth.
return _to_path_component(split_auth_from_netloc(self.netloc)[0])
@property
def file_path(self) -> str:
@@ -7,8 +7,7 @@ from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._internal.exceptions import CommandError
# TODO: add slots=True when Python 3.9 is dropped
@dataclass
@dataclass(slots=True)
class ReleaseControl:
"""Helper for managing which release types can be installed."""
@@ -10,14 +10,12 @@ from dataclasses import dataclass
SCHEME_KEYS = ["platlib", "purelib", "headers", "scripts", "data"]
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class Scheme:
"""A Scheme holds paths which are used as the base directories for
artifacts associated with a Python package.
"""
__slots__ = SCHEME_KEYS
platlib: str
purelib: str
headers: str
@@ -14,14 +14,12 @@ from pip._internal.utils.misc import normalize_path, redact_auth_from_url
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class SearchScope:
"""
Encapsulates the locations that pip is configured to search.
"""
__slots__ = ["find_links", "index_urls", "no_index"]
find_links: list[str]
index_urls: list[str]
no_index: bool
@@ -1,56 +1,36 @@
from __future__ import annotations
from dataclasses import dataclass
from pip._internal.models.format_control import FormatControl
from pip._internal.models.release_control import ReleaseControl
# TODO: This needs Python 3.10's improved slots support for dataclasses
# to be converted into a dataclass.
@dataclass(slots=True)
class SelectionPreferences:
"""
Encapsulates the candidate selection preferences for downloading
and installing files.
"""
__slots__ = [
"allow_yanked",
"release_control",
"format_control",
"prefer_binary",
"ignore_requires_python",
]
:param allow_yanked: Whether files marked as yanked (in the sense
of PEP 592) are permitted to be candidates for install.
:param release_control: A ReleaseControl object or None. Used to control
whether pre-releases are allowed for specific packages.
:param format_control: A FormatControl object or None. Used to control
the selection of source packages / binary packages when consulting
the index and links.
:param prefer_binary: Whether to prefer an old, but valid, binary
dist over a new source dist.
:param ignore_requires_python: Whether to ignore incompatible
"Requires-Python" values in links. Defaults to False.
"""
# Don't include an allow_yanked default value to make sure each call
# site considers whether yanked releases are allowed. This also causes
# that decision to be made explicit in the calling code, which helps
# people when reading the code.
def __init__(
self,
allow_yanked: bool,
release_control: ReleaseControl | None = None,
format_control: FormatControl | None = None,
prefer_binary: bool = False,
ignore_requires_python: bool | None = None,
) -> None:
"""Create a SelectionPreferences object.
:param allow_yanked: Whether files marked as yanked (in the sense
of PEP 592) are permitted to be candidates for install.
:param release_control: A ReleaseControl object or None. Used to control
whether pre-releases are allowed for specific packages.
:param format_control: A FormatControl object or None. Used to control
the selection of source packages / binary packages when consulting
the index and links.
:param prefer_binary: Whether to prefer an old, but valid, binary
dist over a new source dist.
:param ignore_requires_python: Whether to ignore incompatible
"Requires-Python" values in links. Defaults to False.
"""
if ignore_requires_python is None:
ignore_requires_python = False
self.allow_yanked = allow_yanked
self.release_control = release_control
self.format_control = format_control
self.prefer_binary = prefer_binary
self.ignore_requires_python = ignore_requires_python
allow_yanked: bool
release_control: ReleaseControl | None = None
format_control: FormatControl | None = None
prefer_binary: bool = False
ignore_requires_python: bool = False
@@ -6,6 +6,7 @@ providing credentials in the context of network requests.
from __future__ import annotations
import json
import logging
import os
import shutil
@@ -15,7 +16,7 @@ import typing
import urllib.parse
from abc import ABC, abstractmethod
from functools import cache
from os.path import commonprefix
from os.path import commonpath
from pathlib import Path
from typing import Any, NamedTuple
@@ -118,34 +119,48 @@ class KeyRingCliProvider(KeyRingBaseProvider):
self.keyring = cmd
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
# This is the default implementation of keyring.get_credential
# https://github.com/jaraco/keyring/blob/97689324abcf01bd1793d49063e7ca01e03d7d07/keyring/backend.py#L134-L139
if username is not None:
password = self._get_password(url, username)
if password is not None:
return username, password
return None
return self._get_creds(url, username)
def save_auth_info(self, url: str, username: str, password: str) -> None:
return self._set_password(url, username, password)
def _get_password(self, service_name: str, username: str) -> str | None:
"""Mirror the implementation of keyring.get_password using cli"""
def _get_creds(self, service_name: str, username: str | None) -> AuthInfo | None:
"""Mirror the implementation of keyring.get_credential using cli"""
if self.keyring is None:
return None
cmd = [self.keyring, "get", service_name, username]
cmd = [self.keyring, "--mode=creds", "--output=json", "get", service_name]
if username is not None:
cmd.append(username)
env = os.environ.copy()
env["PYTHONIOENCODING"] = "utf-8"
res = subprocess.run(
res = subprocess.run( # noqa: UP022
cmd,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
)
# Detect if the user is running an outdated version of keyring without support
# for querying credentials without username
errs = res.stderr.decode("utf-8")
if (
res.returncode == 2
and "unrecognized arguments" in errs
and "--mode=creds" in errs
):
raise RuntimeError(
"Keyring util is outdated; must be at least version 25.2.1, "
"please upgrade it"
)
if res.returncode:
return None
return res.stdout.decode("utf-8").strip(os.linesep)
data = json.loads(res.stdout.decode("utf-8"))
return (data["username"], data["password"])
def _set_password(self, service_name: str, username: str, password: str) -> None:
"""Mirror the implementation of keyring.set_password using cli"""
@@ -325,12 +340,14 @@ class MultiDomainBasicAuth(AuthBase):
candidates.sort(
reverse=True,
key=lambda candidate: commonprefix(
[
parsed_url.path,
candidate.path,
]
).rfind("/"),
key=lambda candidate: len(
commonpath(
[
parsed_url.path,
candidate.path,
]
)
),
)
return urllib.parse.urlunsplit(candidates[0])
@@ -485,15 +502,17 @@ class MultiDomainBasicAuth(AuthBase):
if resp.status_code != 401:
return resp
username, password = None, None
# Query the keyring for credentials:
if self.use_keyring:
username, password = self._get_new_credentials(
resp.url,
allow_netrc=False,
allow_keyring=True,
)
# Look for credentials for the (possibly redirected) URL. Credentials
# embedded in the URL -- e.g. carried in the ``Location`` of a
# cross-origin redirect, whose ``Authorization`` header requests strips
# -- are always honoured, since recovering them needs no user
# interaction. Keyring is only consulted when it is enabled, because it
# may require interaction and is therefore disabled under --no-input.
username, password = self._get_new_credentials(
resp.url,
allow_netrc=False,
allow_keyring=self.use_keyring,
)
# We are not able to prompt the user so simply return the response
if not self.prompting and not username and not password:
@@ -4,10 +4,10 @@ from __future__ import annotations
import os
import shutil
from collections.abc import Generator
from collections.abc import Callable, Generator
from contextlib import contextmanager
from datetime import datetime
from typing import Any, BinaryIO, Callable
from typing import Any, BinaryIO
from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
from pip._vendor.cachecontrol.caches import SeparateBodyFileCache
@@ -2,6 +2,7 @@
from __future__ import annotations
import contextlib
import email.message
import logging
import mimetypes
@@ -15,12 +16,23 @@ from pip._vendor.requests import PreparedRequest
from pip._vendor.requests.models import Response
from pip._vendor.urllib3 import HTTPResponse as URLlib3Response
from pip._vendor.urllib3._collections import HTTPHeaderDict
from pip._vendor.urllib3.exceptions import ReadTimeoutError
from pip._vendor.urllib3.exceptions import ProtocolError, ReadTimeoutError
from pip._internal.cli.progress_bars import BarType, get_download_progress_renderer
from pip._internal.exceptions import IncompleteDownloadError, NetworkConnectionError
from pip._internal.models.index import PyPI
from pip._internal.models.link import Link
from pip._internal.exceptions import (
ConnectionFailedError,
ConnectionTimeoutError,
IncompleteDownloadError,
NetworkConnectionError,
ProxyConnectionError,
SSLVerificationError,
)
from pip._internal.models.link import (
Link,
PathComponent,
as_path_component,
join_within_directory,
)
from pip._internal.network.cache import SafeFileCache, is_from_cache
from pip._internal.network.session import CacheControlAdapter, PipSession
from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
@@ -31,9 +43,14 @@ logger = logging.getLogger(__name__)
def _get_http_response_size(resp: Response) -> int | None:
try:
return int(resp.headers["content-length"])
size = int(resp.headers["content-length"])
except (ValueError, KeyError, TypeError):
return None
# A negative length would make _FileDownload.is_incomplete() report a
# truncated download as complete, so treat it as unknown instead.
if size < 0:
return None
return size
def _get_http_response_etag_or_last_modified(resp: Response) -> str | None:
@@ -51,10 +68,10 @@ def _log_download(
total_length: int | None,
range_start: int | None = 0,
) -> Iterable[bytes]:
if link.netloc == PyPI.file_storage_domain:
url = link.show_url
else:
if logger.getEffectiveLevel() > logging.INFO:
url = link.url_without_fragment
else:
url = link.show_url
logged_url = redact_auth_from_url(url)
@@ -117,11 +134,14 @@ def parse_content_disposition(content_disposition: str, default_filename: str) -
return filename or default_filename
def _get_http_response_filename(resp: Response, link: Link) -> str:
def _get_http_response_filename(resp: Response, link: Link) -> PathComponent:
"""Get an ideal filename from the given HTTP response, falling back to
the link filename if not provided.
The result is validated as a single path component, so it can be joined onto
a download directory without escaping it.
"""
filename = link.filename # fallback
filename: str = link.filename # fallback
# Have a look at the Content-Disposition header for a better guess
content_disposition = resp.headers.get("content-disposition")
if content_disposition:
@@ -135,7 +155,7 @@ def _get_http_response_filename(resp: Response, link: Link) -> str:
ext = os.path.splitext(resp.url)[1]
if ext:
filename += ext
return filename
return as_path_component(filename)
@dataclass
@@ -188,12 +208,19 @@ class Downloader:
resp = self._http_get(link)
download_size = _get_http_response_size(resp)
filepath = os.path.join(location, _get_http_response_filename(resp, link))
with open(filepath, "wb") as content_file:
download = _FileDownload(link, content_file, download_size)
self._process_response(download, resp)
if download.is_incomplete():
self._attempt_resumes_or_redownloads(download, resp)
filepath = join_within_directory(
location, _get_http_response_filename(resp, link)
)
try:
with open(filepath, "wb") as content_file:
download = _FileDownload(link, content_file, download_size)
self._process_response(download, resp)
if download.is_incomplete():
self._attempt_resumes_or_redownloads(download, resp)
except IncompleteDownloadError:
with contextlib.suppress(OSError):
os.remove(filepath)
raise
content_type = resp.headers.get("Content-Type", "")
return filepath, content_type
@@ -210,12 +237,12 @@ class Downloader:
try:
for chunk in chunks:
download.write_chunk(chunk)
except ReadTimeoutError as e:
except (ReadTimeoutError, ProtocolError) as e:
# If the download size is not known, then give up downloading the file.
if download.size is None:
raise e
logger.warning("Connection timed out while downloading.")
logger.warning("Connection interrupted while downloading.")
def _attempt_resumes_or_redownloads(
self, download: _FileDownload, first_resp: Response
@@ -242,14 +269,38 @@ class Downloader:
download.reset_file()
download.size = _get_http_response_size(resume_resp)
first_resp = resume_resp
else:
# If the resume request starts at the wrong location, fail
# outright since the server is misbehaving.
content_range = resume_resp.headers.get("Content-Range", "")
resumed_at = content_range.lower().partition("bytes ")[2]
resumed_at = resumed_at.partition("-")[0]
if resumed_at and resumed_at != str(download.bytes_received):
raise IncompleteDownloadError(download)
self._process_response(download, resume_resp)
except (ConnectionError, ReadTimeoutError, OSError):
except (
ConnectionFailedError,
ConnectionTimeoutError,
ProxyConnectionError,
SSLVerificationError,
ReadTimeoutError,
ProtocolError,
OSError,
):
# The error handling here is tricky, a few notes:
#
# - The diagnostic connection errors are raised by our custom
# requests.request() connection exception handler.
# - ProtocolError is raised by urllib3 when the returned data length
# doesn't match the response Content-Length.
# - ReadTimeoutError / ProtocolError come straight from urllib3 (via
# process_response) and aren't caught by our connection exception
# handler since they occur while *streaming* a response.
continue
# No more resume attempts. Raise an error if the download is still incomplete.
if download.is_incomplete():
os.remove(download.output_file.name)
raise IncompleteDownloadError(download)
# If we successfully completed the download via resume, manually cache it
@@ -22,8 +22,6 @@ from collections.abc import Generator, Mapping, Sequence
from typing import (
TYPE_CHECKING,
Any,
Optional,
Union,
)
from pip._vendor import requests, urllib3
@@ -36,27 +34,33 @@ from pip._vendor.urllib3.connectionpool import ConnectionPool
from pip._vendor.urllib3.exceptions import InsecureRequestWarning
from pip import __version__
from pip._internal.exceptions import SSLMissingError
from pip._internal.metadata import get_default_environment
from pip._internal.models.link import Link
from pip._internal.network.auth import MultiDomainBasicAuth
from pip._internal.network.cache import SafeFileCache
from pip._internal.network.utils import raise_connection_error
# Import ssl from compat so the initial import occurs in only one place.
from pip._internal.utils.compat import has_tls
from pip._internal.utils.glibc import libc_ver
from pip._internal.utils.misc import build_url_from_netloc, parse_netloc
from pip._internal.utils.misc import (
build_url_from_netloc,
looks_like_ci,
parse_netloc,
redact_auth_from_url,
)
from pip._internal.utils.urls import url_to_path
if TYPE_CHECKING:
from ssl import SSLContext
from pip._vendor.urllib3 import ProxyManager
from pip._vendor.urllib3.poolmanager import PoolManager
logger = logging.getLogger(__name__)
SecureOrigin = tuple[str, str, Optional[Union[int, str]]]
SecureOrigin = tuple[str, str, int | str | None]
# Ignore warning raised when using --trusted-host.
@@ -76,35 +80,6 @@ SECURE_ORIGINS: list[SecureOrigin] = [
]
# These are environment variables present when running under various
# CI systems. For each variable, some CI systems that use the variable
# are indicated. The collection was chosen so that for each of a number
# of popular systems, at least one of the environment variables is used.
# This list is used to provide some indication of and lower bound for
# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive.
# For more background, see: https://github.com/pypa/pip/issues/5499
CI_ENVIRONMENT_VARIABLES = (
# Azure Pipelines
"BUILD_BUILDID",
# Jenkins
"BUILD_ID",
# AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI
"CI",
# Explicit environment variable.
"PIP_IS_CI",
)
def looks_like_ci() -> bool:
"""
Return whether it looks like pip is running under CI.
"""
# We don't use the method of checking for a tty (e.g. using isatty())
# because some CI systems mimic a tty (e.g. Travis CI). Thus that
# method doesn't provide definitive information in either direction.
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
@functools.lru_cache(maxsize=1)
def user_agent() -> str:
"""
@@ -212,7 +187,7 @@ class LocalFSAdapter(BaseAdapter):
self,
request: PreparedRequest,
stream: bool = False,
timeout: float | tuple[float, float] | tuple[float, None] | None = None,
timeout: float | tuple[float | None, float | None] | None = None,
verify: bool | str = True,
cert: bytes | str | tuple[bytes | str, bytes | str] | None = None,
proxies: Mapping[str, str] | None = None,
@@ -264,9 +239,11 @@ class _SSLContextAdapterMixin:
self,
*,
ssl_context: SSLContext | None = None,
use_ssl_context_for_proxy: bool = True,
**kwargs: Any,
) -> None:
self._ssl_context = ssl_context
self._use_ssl_context_for_proxy = use_ssl_context_for_proxy
super().__init__(**kwargs)
def init_poolmanager(
@@ -275,10 +252,10 @@ class _SSLContextAdapterMixin:
maxsize: int,
block: bool = DEFAULT_POOLBLOCK,
**pool_kwargs: Any,
) -> PoolManager:
) -> None:
if self._ssl_context is not None:
pool_kwargs.setdefault("ssl_context", self._ssl_context)
return super().init_poolmanager( # type: ignore[misc, no-any-return]
super().init_poolmanager( # type: ignore[misc]
connections=connections,
maxsize=maxsize,
block=block,
@@ -290,7 +267,10 @@ class _SSLContextAdapterMixin:
# context here too. https://github.com/pypa/pip/issues/13288
if self._ssl_context is not None:
proxy_kwargs.setdefault("ssl_context", self._ssl_context)
return super().proxy_manager_for(proxy, **proxy_kwargs) # type: ignore[misc, no-any-return]
if self._use_ssl_context_for_proxy:
# Leave this unset when urllib3 must apply a custom proxy CA bundle.
proxy_kwargs.setdefault("proxy_ssl_context", self._ssl_context)
return super().proxy_manager_for(proxy, **proxy_kwargs) # type: ignore[misc]
class HTTPAdapter(_SSLContextAdapterMixin, _BaseHTTPAdapter):
@@ -335,6 +315,8 @@ class PipSession(requests.Session):
trusted_hosts: Sequence[str] = (),
index_urls: list[str] | None = None,
ssl_context: SSLContext | None = None,
use_ssl_context_for_proxy: bool = True,
refresh_package: set[str] | None = None,
**kwargs: Any,
) -> None:
"""
@@ -346,11 +328,19 @@ class PipSession(requests.Session):
# Namespace the attribute with "pip_" just in case to prevent
# possible conflicts with the base class.
self.pip_trusted_origins: list[tuple[str, int | None]] = []
self.pip_proxy = None
# "" disables proxying; None means no --proxy was given.
self.pip_proxy: str | None = None
self.pip_no_proxy_env = False
self.refresh_package: set[str] = refresh_package or set()
# Attach our User Agent to the request
self.headers["User-Agent"] = user_agent()
# Pin Accept-Encoding so it doesn't vary with zstd availability (Python
# 3.14+ or backports.zstd); a varying value misses the cache for "Vary:
# Accept-Encoding" responses shared across interpreters (pypa/pip#13979).
self.headers["Accept-Encoding"] = "gzip, deflate"
# Attach our Authentication handler to the session
self.auth: MultiDomainBasicAuth = MultiDomainBasicAuth(index_urls=index_urls)
@@ -392,13 +382,18 @@ class PipSession(requests.Session):
cache=SafeFileCache(cache),
max_retries=retries,
ssl_context=ssl_context,
use_ssl_context_for_proxy=use_ssl_context_for_proxy,
)
self._trusted_host_adapter = InsecureCacheControlAdapter(
cache=SafeFileCache(cache),
max_retries=retries,
)
else:
secure_adapter = HTTPAdapter(max_retries=retries, ssl_context=ssl_context)
secure_adapter = HTTPAdapter(
max_retries=retries,
ssl_context=ssl_context,
use_ssl_context_for_proxy=use_ssl_context_for_proxy,
)
self._trusted_host_adapter = insecure_adapter
self.mount("https://", secure_adapter)
@@ -522,6 +517,23 @@ class PipSession(requests.Session):
return False
def get_redirect_target(self, resp: Response) -> str | None:
target = super().get_redirect_target(resp)
if target is None:
return None
# Redirecting to a file:// URL doesn't make much sense and
# shouldn't be allowed.
scheme = urllib.parse.urlparse(target).scheme
if scheme and scheme.lower() not in ("http", "https"):
logger.warning(
"Not following redirect from %s to %s: a redirect to a non-http(s)"
" location is not allowed.",
redact_auth_from_url(resp.url),
redact_auth_from_url(target),
)
return None
return target
def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Response: # type: ignore[override]
# Allow setting a default timeout on a session
kwargs.setdefault("timeout", self.timeout)
@@ -529,4 +541,15 @@ class PipSession(requests.Session):
kwargs.setdefault("proxies", self.proxies)
# Dispatch the actual request
return super().request(method, url, *args, **kwargs)
try:
return super().request(method, url, *args, **kwargs)
except (requests.ConnectionError, requests.Timeout) as e:
request = getattr(e, "request", None)
failed_url = getattr(request, "url", None) or url
raise_connection_error(e, url=failed_url, timeout=kwargs["timeout"])
except ImportError as e:
if "ssl" in str(e).lower():
# Unfortunately, if this TLS error was the result of a redirect from
# a HTTP to a HTTPS url, we don't know what the final url was.
raise SSLMissingError(redact_auth_from_url(url))
raise
@@ -1,8 +1,23 @@
from collections.abc import Generator
from typing import Literal, NoReturn, TypeAlias, cast
from urllib.parse import urlsplit
from pip._vendor import requests, urllib3
from pip._vendor.requests.models import Response
from pip._vendor.urllib3.exceptions import TimeoutStateError
from pip._internal.exceptions import NetworkConnectionError
from pip._internal.exceptions import (
ConnectionFailedError,
ConnectionTimeoutError,
NetworkConnectionError,
ProxyConnectionError,
SSLVerificationError,
)
from pip._internal.utils.misc import redact_auth_from_url
TimeoutValue: TypeAlias = (
float | tuple[float | None, float | None] | urllib3.util.Timeout | None
)
# The following comments and HTTP headers were originally added by
# Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
@@ -96,3 +111,96 @@ def response_chunks(
if not chunk:
break
yield chunk
def _parse_timeout(timeout: TimeoutValue, kind: Literal["connect", "read"]) -> float:
connect_timeout: float | None
read_timeout: float | None
if isinstance(timeout, tuple):
connect_timeout, read_timeout = timeout
elif isinstance(timeout, urllib3.util.Timeout):
connect_timeout = cast(float | None, timeout.connect_timeout)
try:
read_timeout = timeout.read_timeout
except TimeoutStateError:
read_timeout = None
else:
connect_timeout = read_timeout = timeout
if kind == "connect":
assert connect_timeout is not None
return connect_timeout
else:
assert read_timeout is not None
return read_timeout
def _raise_timeout_error(
reason: urllib3.exceptions.TimeoutError,
url: str,
host: str,
timeout: TimeoutValue,
) -> NoReturn:
if isinstance(reason, urllib3.exceptions.ConnectTimeoutError):
raise ConnectionTimeoutError(
url, host, kind="connect", timeout=_parse_timeout(timeout, "connect")
)
else:
raise ConnectionTimeoutError(
url, host, kind="read", timeout=_parse_timeout(timeout, "read")
)
def raise_connection_error(
error: requests.ConnectionError | requests.Timeout,
*,
url: str,
timeout: TimeoutValue,
) -> NoReturn:
"""Raise a specific error for a given connection error, if possible.
Note: requests.ConnectionError is the parent class of
requests.ProxyError, requests.SSLError, and requests.ConnectTimeout
so these errors are also handled here.
"""
url = redact_auth_from_url(url)
raw_hostname = urlsplit(url).hostname or urlsplit(url).netloc
reason = error.args[0] if error.args else error
# NewConnectionError is a subclass of TimeoutError for some reason...
if isinstance(reason, urllib3.exceptions.TimeoutError) and not isinstance(
reason, urllib3.exceptions.NewConnectionError
):
# A bare timeout error can occur during non-streamed responses. Don't
# ask me how.
_raise_timeout_error(reason, url, raw_hostname, timeout)
if isinstance(reason, urllib3.exceptions.SSLError):
# A bare SSL error can occur during non-streamed responses, after the
# initial connection and TLS handshake have completed.
raise SSLVerificationError(url, raw_hostname, reason)
# At this point, all errors should be wrapped in MaxRetryError.
if not isinstance(reason, urllib3.exceptions.MaxRetryError):
raise ConnectionFailedError(url, raw_hostname, reason)
max_retry_error = reason
assert isinstance(max_retry_error.pool, urllib3.connectionpool.HTTPConnectionPool)
host = max_retry_error.pool.host
proxy = max_retry_error.pool.proxy
# Narrow the reason further to the specific error from the last retry.
reason = max_retry_error.reason
if isinstance(reason, urllib3.exceptions.SSLError):
raise SSLVerificationError(url, host, reason)
if isinstance(reason, urllib3.exceptions.TimeoutError) and not isinstance(
reason, urllib3.exceptions.NewConnectionError
):
_raise_timeout_error(reason, url, host, timeout)
if isinstance(reason, urllib3.exceptions.ProxyError):
assert proxy is not None
raise ProxyConnectionError(url, redact_auth_from_url(str(proxy)), reason)
# Unknown error, give up and raise a generic error.
raise ConnectionFailedError(
url, host, reason if isinstance(reason, Exception) else max_retry_error
)
@@ -3,12 +3,11 @@
from __future__ import annotations
import logging
from collections.abc import Generator, Iterable
from collections.abc import Callable, Generator, Iterable
from contextlib import suppress
from email.parser import Parser
from functools import reduce
from typing import (
Callable,
NamedTuple,
)
@@ -168,7 +167,7 @@ def _create_whitelist(
continue
for req in package_set[package_name].dependencies:
if canonicalize_name(req.name) in packages_affected:
if canonicalize_name(req.name) in would_be_installed:
packages_affected.add(package_name)
break
@@ -15,26 +15,25 @@ import sys
import textwrap
import warnings
from base64 import urlsafe_b64encode
from collections.abc import Generator, Iterable, Iterator, Sequence
from collections.abc import Callable, Generator, Iterable, Iterator, Sequence
from email.message import Message
from itertools import chain, filterfalse, starmap
from pathlib import Path
from typing import (
IO,
Any,
BinaryIO,
Callable,
NewType,
Protocol,
Union,
cast,
)
from zipfile import ZipFile, ZipInfo
from pip._vendor.distlib.scripts import ScriptMaker
from pip._vendor.distlib.util import get_export_entry
from pip._vendor.distlib.util import FileOperator, get_export_entry
from pip._vendor.packaging.utils import canonicalize_name
from pip._internal.exceptions import InstallationError
from pip._internal.exceptions import BlockedImportError, InstallationError
from pip._internal.locations import get_major_minor_version
from pip._internal.metadata import (
BaseDistribution,
@@ -59,6 +58,10 @@ class File(Protocol):
dest_path: str
changed: bool
@property
def written(self) -> bool:
pass
def save(self) -> None:
pass
@@ -66,7 +69,7 @@ class File(Protocol):
logger = logging.getLogger(__name__)
RecordPath = NewType("RecordPath", str)
InstalledCSVRow = tuple[RecordPath, str, Union[int, str]]
InstalledCSVRow = tuple[RecordPath, str, int | str]
def rehash(path: str, blocksize: int = 1 << 20) -> tuple[str, str]:
@@ -127,27 +130,33 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> str | None:
return None
# Group scripts by the path they were installed in
grouped_by_dir: dict[str, set[str]] = collections.defaultdict(set)
grouped_by_dir: dict[Path, set[str]] = collections.defaultdict(set)
for destfile in scripts:
parent_dir = os.path.dirname(destfile)
script_name = os.path.basename(destfile)
dest_path = Path(destfile)
parent_dir = dest_path.parent.resolve()
script_name = dest_path.name
grouped_by_dir[parent_dir].add(script_name)
# We don't want to warn for directories that are on PATH.
not_warn_dirs = [
os.path.normcase(os.path.normpath(i)).rstrip(os.sep)
for i in os.environ.get("PATH", "").split(os.pathsep)
]
# If an executable sits with sys.executable, we don't warn for it.
# This covers the case of venv invocations without activating the venv.
not_warn_dirs.append(
os.path.normcase(os.path.normpath(os.path.dirname(sys.executable)))
)
warn_for: dict[str, set[str]] = {
executable_dir = Path(sys.executable).parent.resolve()
path_entries = os.environ.get("PATH", "").split(os.pathsep)
# We don't want to warn for directories that are on PATH.
warn_for: dict[Path, set[str]] = {
parent_dir: scripts
for parent_dir, scripts in grouped_by_dir.items()
if os.path.normcase(os.path.normpath(parent_dir)) not in not_warn_dirs
if parent_dir != executable_dir and str(parent_dir) not in path_entries
}
# Resolving PATH entries can be slow, so only do it for what's left.
if warn_for:
not_warn_dirs = {Path(i).resolve() for i in path_entries}
warn_for = {
parent_dir: scripts
for parent_dir, scripts in warn_for.items()
if parent_dir not in not_warn_dirs
}
if not warn_for:
return None
@@ -176,9 +185,7 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> str | None:
msg_lines.append(last_line_fmt.format("these directories"))
# Add a note if any directory starts with ~
warn_for_tilde = any(
i[0] == "~" for i in os.environ.get("PATH", "").split(os.pathsep) if i
)
warn_for_tilde = any(i[0] == "~" for i in path_entries if i)
if warn_for_tilde:
tilde_warning_msg = (
"NOTE: The current PATH contains path(s) starting with `~`, "
@@ -345,6 +352,7 @@ class ZipBackedFile:
self.dest_path = dest_path
self._zip_file = zip_file
self.changed = False
self.written = False
def _getinfo(self) -> ZipInfo:
return self._zip_file.getinfo(self.src_record_path)
@@ -366,6 +374,7 @@ class ZipBackedFile:
# optimization: the file is created by open(),
# skip the decompression when there is 0 bytes to decompress.
with open(self.dest_path, "wb") as dest:
self.written = True
if zipinfo.file_size > 0:
with self._zip_file.open(zipinfo) as f:
blocksize = min(zipinfo.file_size, 1024 * 1024)
@@ -382,6 +391,10 @@ class ScriptFile:
self.dest_path = self._file.dest_path
self.changed = False
@property
def written(self) -> bool:
return self._file.written
def save(self) -> None:
self._file.save()
self.changed = fix_script(self.dest_path)
@@ -397,32 +410,171 @@ class MissingCallableSuffix(InstallationError):
)
def _raise_for_invalid_entrypoint(specification: str) -> None:
def _raise_for_invalid_entrypoint(specification: str, scripts_dir: str) -> None:
entry = get_export_entry(specification)
if entry is not None and entry.suffix is None:
if entry is None:
return
if entry.suffix is None:
raise MissingCallableSuffix(str(entry))
# distlib joins the entry point name onto the scripts directory, so a name
# with path separators or ``..`` components can resolve elsewhere. The script
# must resolve to a path strictly inside the scripts directory.
dest = os.path.join(scripts_dir, entry.name)
resolves_to_scripts_dir = os.path.abspath(dest) == os.path.abspath(scripts_dir)
if resolves_to_scripts_dir or not is_within_directory(scripts_dir, dest):
raise InstallationError(
f"Invalid script entry point name {entry.name!r}: the script "
f"would be installed outside the scripts directory ({scripts_dir})."
)
class PipScriptMaker(ScriptMaker):
# Override distlib's default script template with one that
# doesn't import `re` module, allowing scripts to load faster.
script_template = textwrap.dedent(
"""\
script_template = textwrap.dedent("""\
import sys
from %(module)s import %(import_name)s
if __name__ == '__main__':
sys.argv[0] = sys.argv[0].removesuffix('.exe')
sys.exit(%(func)s())
"""
)
""")
def make(
self, specification: str, options: dict[str, Any] | None = None
) -> list[str]:
_raise_for_invalid_entrypoint(specification)
_raise_for_invalid_entrypoint(specification, self.target_dir)
return super().make(specification, options)
class _WheelInstallRecord:
"""Track files and metadata needed to recover an interrupted installation."""
def __init__(self, info_dir: str, lib_dir: str) -> None:
self.info_dir = info_dir
self.lib_dir = lib_dir
self.installed: dict[RecordPath, RecordPath] = {}
self.generated: list[str] = []
self.script_fileop = FileOperator()
self.script_fileop.record = True
self.started = False
def written_paths(self) -> set[str]:
"""Return the files written so far, including partial copies and scripts."""
paths = {os.path.join(self.lib_dir, path) for path in self.installed.values()}
paths.update(self.generated)
paths.update(self.script_fileop.files_written)
return paths
@contextlib.contextmanager
def _generate_file(
path: str, file_mode: int, **kwargs: Any
) -> Generator[BinaryIO, None, None]:
"""Write metadata atomically with the requested permissions."""
temporary_path = None
try:
with adjacent_tmp_file(path, **kwargs) as f:
temporary_path = f.name
yield f
os.chmod(f.name, file_mode)
replace(f.name, path)
except BaseException:
if temporary_path is not None:
with contextlib.suppress(FileNotFoundError):
os.unlink(temporary_path)
raise
def _write_recovery_record(wheel_zip: ZipFile, record: _WheelInstallRecord) -> None:
"""Write readable metadata and an uninstall record for files already written."""
info_dir, lib_dir = record.info_dir, record.lib_dir
file_mode = 0o666 & ~current_umask()
recovery_paths = {
_fs_to_record_path(path, lib_dir) for path in record.written_paths()
}
dest_info_dir = os.path.join(lib_dir, info_dir)
ensure_dir(dest_info_dir)
metadata_path = os.path.join(dest_info_dir, "METADATA")
with _generate_file(metadata_path, file_mode) as metadata_file:
metadata_file.write(wheel_zip.read(f"{info_dir}/METADATA"))
recovery_paths.add(_fs_to_record_path(metadata_path, lib_dir))
# Uninstall must not infer wrappers that were never generated.
with contextlib.suppress(FileNotFoundError):
os.unlink(os.path.join(dest_info_dir, "entry_points.txt"))
record_path = os.path.join(dest_info_dir, "RECORD")
recovery_paths.add(_fs_to_record_path(record_path, lib_dir))
with _generate_file(record_path, file_mode, **csv_io_kwargs("w")) as record_file:
writer = csv.writer(cast("IO[str]", record_file))
writer.writerows((path, "", "") for path in sorted(recovery_paths))
@contextlib.contextmanager
def _record_wheel_installation(
wheel_zip: ZipFile, name: str, scheme: Scheme, *, rollback_on_error: bool = False
) -> Generator[_WheelInstallRecord, None, None]:
"""Provide the file journal and recover writes interrupted by blocked imports."""
info_dir, metadata = parse_wheel(wheel_zip, name)
lib_dir = scheme.purelib if wheel_root_is_purelib(metadata) else scheme.platlib
record = _WheelInstallRecord(info_dir, lib_dir)
try:
yield record
except BlockedImportError as error:
# A blocked preparation step must leave an existing install alone.
if not record.started:
raise
if rollback_on_error:
# Remove replacement files before the previous installation is restored.
try:
for path in record.written_paths():
with contextlib.suppress(FileNotFoundError):
os.unlink(path)
# Empty dist-info directories still look like installed distributions.
for directory, _, _ in os.walk(
os.path.join(lib_dir, info_dir), topdown=False
):
with contextlib.suppress(OSError):
os.rmdir(directory)
except (OSError, BlockedImportError) as cleanup_error:
raise BlockedImportError(
f"Could not remove incomplete installation of {name!r}: "
f"{cleanup_error}\n{error}"
) from cleanup_error
raise
try:
_write_recovery_record(wheel_zip, record)
except (OSError, BlockedImportError) as recovery_error:
raise BlockedImportError(
f"Could not write recovery metadata for {name!r} at "
f"{os.path.join(lib_dir, info_dir)!r}: {recovery_error}\n{error}"
) from recovery_error
raise BlockedImportError(
f"Installation of {name!r} was interrupted and the package may be "
f"corrupt. Run 'pip uninstall {name}' to remove it. {error}"
) from error
def _compile_wheel_file(path: str) -> bool:
"""Skip optional compilation when it requires a blocked or unavailable import."""
try:
return compileall.compile_file(path, force=True, quiet=True)
except (BlockedImportError, ImportError) as error:
logger.debug("Skipping bytecode compilation for %s: %s", path, error)
return False
def _install_wheel( # noqa: C901, PLR0915 function is too long
name: str,
wheel_zip: ZipFile,
@@ -432,36 +584,33 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
warn_script_location: bool = True,
direct_url: DirectUrl | None = None,
requested: bool = False,
script_executable: str | None = None,
*,
record: _WheelInstallRecord,
) -> None:
"""Install a wheel.
:param name: Name of the project to install
:param wheel_zip: open ZipFile for wheel being installed
:param scheme: Distutils scheme dictating the install directories
:param req_description: String used in place of the requirement, for
logging
:param pycompile: Whether to byte-compile installed Python files
:param warn_script_location: Whether to check that scripts are installed
into a directory on PATH
:param script_executable: Python executable to use for console scripts
:raises UnsupportedWheel:
* when the directory holds an unpacked wheel with incompatible
Wheel-Version
* when the .dist-info dir does not match the wheel
"""
info_dir, metadata = parse_wheel(wheel_zip, name)
if wheel_root_is_purelib(metadata):
lib_dir = scheme.purelib
else:
lib_dir = scheme.platlib
info_dir, lib_dir = record.info_dir, record.lib_dir
# Record details of the files moved
# installed = files copied from the wheel to the destination
# changed = files changed while installing (scripts #! line typically)
# generated = files newly generated during the install (script wrappers)
installed: dict[RecordPath, RecordPath] = {}
# generated = scripts and installation metadata generated during the install
installed = record.installed
changed: set[RecordPath] = set()
generated: list[str] = []
generated = record.generated
def record_installed(
srcfile: RecordPath, destfile: str, modified: bool = False
@@ -582,6 +731,8 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
script_scheme_files = map(ScriptFile, script_scheme_files)
files = chain(files, script_scheme_files)
# File saves may unlink old files before creating replacements.
record.started = True
existing_parents = set()
for file in files:
# directory creation is lazy and after file filtering
@@ -591,7 +742,14 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
if parent_dir not in existing_parents:
ensure_dir(parent_dir)
existing_parents.add(parent_dir)
file.save()
try:
file.save()
except BlockedImportError:
if file.written:
record_installed(file.src_record_path, file.dest_path, True)
raise
record_installed(file.src_record_path, file.dest_path, file.changed)
def pyc_source_file_paths() -> Generator[str, None, None]:
@@ -619,7 +777,7 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
for path in pyc_source_file_paths():
success = compileall.compile_file(path, force=True, quiet=True)
success = _compile_wheel_file(path)
if success:
pyc_path = pyc_output_path(path)
assert os.path.exists(pyc_path)
@@ -629,7 +787,14 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
record_installed(pyc_record_path, pyc_path)
logger.debug(stdout.getvalue())
maker = PipScriptMaker(None, scheme.scripts)
# Track scripts even if generation fails before returning their paths.
maker = PipScriptMaker(None, scheme.scripts, fileop=record.script_fileop)
# Embed the target environment's interpreter in console-script launchers
# rather than the one running pip, so an in-process install into another
# environment (e.g. a venv build environment) produces working launchers.
if script_executable is not None:
maker.executable = script_executable # type: ignore # it's untyped in distlib
# Ensure old scripts are overwritten.
# See https://github.com/pypa/pip/issues/1800
@@ -661,26 +826,18 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
logger.warning(msg)
generated_file_mode = 0o666 & ~current_umask()
@contextlib.contextmanager
def _generate_file(path: str, **kwargs: Any) -> Generator[BinaryIO, None, None]:
with adjacent_tmp_file(path, **kwargs) as f:
yield f
os.chmod(f.name, generated_file_mode)
replace(f.name, path)
dest_info_dir = os.path.join(lib_dir, info_dir)
# Record pip as the installer
installer_path = os.path.join(dest_info_dir, "INSTALLER")
with _generate_file(installer_path) as installer_file:
with _generate_file(installer_path, generated_file_mode) as installer_file:
installer_file.write(b"pip\n")
generated.append(installer_path)
# Record the PEP 610 direct URL reference
if direct_url is not None:
direct_url_path = os.path.join(dest_info_dir, DIRECT_URL_METADATA_NAME)
with _generate_file(direct_url_path) as direct_url_file:
with _generate_file(direct_url_path, generated_file_mode) as direct_url_file:
direct_url_file.write(direct_url.to_json().encode("utf-8"))
generated.append(direct_url_path)
@@ -694,9 +851,10 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
record_text = distribution.read_text("RECORD")
record_rows = list(csv.reader(record_text.splitlines()))
# RECORD generation consumes its input; recovery still needs the journal.
rows = get_csv_rows_for_installed(
record_rows,
installed=installed,
installed=installed.copy(),
changed=changed,
generated=generated,
lib_dir=lib_dir,
@@ -705,7 +863,9 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
# Record details of all files installed
record_path = os.path.join(dest_info_dir, "RECORD")
with _generate_file(record_path, **csv_io_kwargs("w")) as record_file:
with _generate_file(
record_path, generated_file_mode, **csv_io_kwargs("w")
) as record_file:
# Explicitly cast to typing.IO[str] as a workaround for the mypy error:
# "writer" has incompatible type "BinaryIO"; expected "_Writer"
writer = csv.writer(cast("IO[str]", record_file))
@@ -730,9 +890,17 @@ def install_wheel(
warn_script_location: bool = True,
direct_url: DirectUrl | None = None,
requested: bool = False,
script_executable: str | None = None,
*,
rollback_on_error: bool = False,
) -> None:
with ZipFile(wheel_path, allowZip64=True) as z:
with req_error_context(req_description):
with (
req_error_context(req_description),
_record_wheel_installation(
z, name, scheme, rollback_on_error=rollback_on_error
) as record,
):
_install_wheel(
name=name,
wheel_zip=z,
@@ -742,4 +910,6 @@ def install_wheel(
warn_script_location=warn_script_location,
direct_url=direct_url,
requested=requested,
script_executable=script_executable,
record=record,
)
@@ -12,9 +12,10 @@ from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.utils import canonicalize_name
from pip._internal.build_env import BuildEnvironmentInstaller
from pip._internal.build_env import BuildEnvironmentInstaller, BuildIsolationMode
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.distributions.installed import InstalledDistribution
from pip._internal.exceptions import (
@@ -23,13 +24,16 @@ from pip._internal.exceptions import (
HashUnpinned,
InstallationError,
MetadataInconsistent,
MetadataInvalid,
NetworkConnectionError,
SidecarMetadataInconsistent,
VcsHashUnsupported,
)
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution, get_metadata_distribution
from pip._internal.models.direct_url import ArchiveInfo
from pip._internal.models.link import Link
from pip._internal.metadata.direct_references import check_pypi_direct_references
from pip._internal.models.direct_url import ArchiveInfo, DirectUrl
from pip._internal.models.link import Link, join_within_directory
from pip._internal.models.wheel import Wheel
from pip._internal.network.download import Downloader
from pip._internal.network.lazy_wheel import (
@@ -52,6 +56,7 @@ from pip._internal.utils.misc import (
hide_url,
redact_auth_from_requirement,
)
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.unpacking import unpack_file
from pip._internal.vcs import vcs
@@ -66,8 +71,9 @@ def _get_prepared_distribution(
req: InstallRequirement,
build_tracker: BuildTracker,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
build_isolation: BuildIsolationMode,
check_build_deps: bool,
allow_editables: bool,
) -> BaseDistribution:
"""Prepare a distribution for installation."""
abstract_dist = make_distribution_for_install_requirement(req)
@@ -75,11 +81,22 @@ def _get_prepared_distribution(
if tracker_id is not None:
with build_tracker.track(req, tracker_id):
abstract_dist.prepare_distribution_metadata(
build_env_installer, build_isolation, check_build_deps
build_env_installer, build_isolation, check_build_deps, allow_editables
)
return abstract_dist.get_metadata_distribution()
def _check_linked_requirement_metadata(
req: InstallRequirement,
distribution: BaseDistribution,
) -> None:
"""Apply source-dependent checks to prepared metadata."""
source_link = req.source_link
if source_link is None:
raise InstallationError(f"Cannot prepare {req}: the source link is missing.")
check_pypi_direct_references(distribution, source_link)
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
vcs_backend = vcs.get_backend_for_scheme(link.scheme)
assert vcs_backend is not None
@@ -201,7 +218,7 @@ def _check_download_dir(
"""Check download_dir for previously downloaded file with correct hash
If a correct file is found return its path else None
"""
download_path = os.path.join(download_dir, link.filename)
download_path = join_within_directory(download_dir, link.filename)
if not os.path.exists(download_path):
return None
@@ -222,6 +239,91 @@ def _check_download_dir(
return download_path
def _canonical_requires(
req: InstallRequirement, dist: BaseDistribution, source: str
) -> frozenset[Requirement]:
"""Return the canonicalized ``Requires-Dist`` entries of ``dist``.
``source`` describes which metadata file ``dist`` was parsed from, for
use in error messages.
"""
canonical = set()
for raw in dist.iter_raw_dependencies():
try:
# strip() because a folded metadata header may be returned
# with a leading newline; iter_dependencies() strips for the
# same reason.
canonical.add(get_requirement(raw.strip()))
except InvalidRequirement as e:
raise MetadataInvalid(req, f"Requires-Dist in {source}: {e}")
return frozenset(canonical)
def _check_sidecar_matches_wheel(
req: InstallRequirement,
sidecar_dist: BaseDistribution,
wheel_dist: BaseDistribution,
) -> None:
"""Check that a .metadata-based distribution matches the wheel's METADATA.
Compare ``Name``, ``Version``, ``Requires-Dist``, ``Requires-Python``
and ``Provides-Extra`` between the two and abort the install on any
mismatch as PEP 658 requires the metadata files "MUST be identical".
While the PEP doesn't mandate that consumers enforce the identical
requirement, it's good nonetheless to check to prevent confusing
behaviour when an index misbehaves.
Also note for name and version, pip usually rejects wheels if they're
inconsistent already. Checking them again here is purely defensive.
"""
sidecar_name = canonicalize_name(sidecar_dist.raw_name)
wheel_name = canonicalize_name(wheel_dist.raw_name)
if sidecar_name != wheel_name:
raise SidecarMetadataInconsistent(req, "Name", sidecar_name, wheel_name)
if sidecar_dist.version != wheel_dist.version:
raise SidecarMetadataInconsistent(
req,
"Version",
str(sidecar_dist.version),
str(wheel_dist.version),
)
# For multi-use fields, only report the symmetric difference to avoid
# unnecessarily flagging matching values.
sidecar_requires = _canonical_requires(
req, sidecar_dist, "the PEP 658 .metadata file"
)
wheel_requires = _canonical_requires(req, wheel_dist, "the wheel's METADATA")
if sidecar_requires != wheel_requires:
raise SidecarMetadataInconsistent(
req,
"Requires-Dist",
", ".join(sorted(map(str, sidecar_requires - wheel_requires))),
", ".join(sorted(map(str, wheel_requires - sidecar_requires))),
)
if sidecar_dist.requires_python != wheel_dist.requires_python:
raise SidecarMetadataInconsistent(
req,
"Requires-Python",
str(sidecar_dist.requires_python),
str(wheel_dist.requires_python),
)
sidecar_extras = frozenset(sidecar_dist.iter_provided_extras())
wheel_extras = frozenset(wheel_dist.iter_provided_extras())
if sidecar_extras != wheel_extras:
raise SidecarMetadataInconsistent(
req,
"Provides-Extra",
", ".join(sorted(sidecar_extras - wheel_extras)),
", ".join(sorted(wheel_extras - sidecar_extras)),
)
class RequirementPreparer:
"""Prepares a Requirement"""
@@ -231,7 +333,7 @@ class RequirementPreparer:
build_dir: str,
download_dir: str | None,
src_dir: str,
build_isolation: bool,
build_isolation: BuildIsolationMode,
build_isolation_installer: BuildEnvironmentInstaller,
check_build_deps: bool,
build_tracker: BuildTracker,
@@ -243,6 +345,7 @@ class RequirementPreparer:
lazy_wheel: bool,
verbosity: int,
legacy_resolver: bool,
allow_editables: bool,
) -> None:
super().__init__()
@@ -285,6 +388,10 @@ class RequirementPreparer:
# Previous "header" printed for a link-based InstallRequirement
self._previous_requirement_header = ("", "")
# Do we allow preparing editable wheels?
# When using pip wheel, editables must still produce regular wheels
self.allow_editables = allow_editables
def _log_preparing_link(self, req: InstallRequirement) -> None:
"""Provide context for the requirement being prepared."""
if req.link.is_file and not req.is_wheel_from_cache:
@@ -342,7 +449,8 @@ class RequirementPreparer:
# (For example, we can raise VcsHashUnsupported for a VCS URL
# rather than HashMissing.)
if not self.require_hashes:
return req.hashes(trust_internet=True)
# An index-provided hash must not override one supplied by the user.
return req.hashes(trust_internet=not req.has_hash_options)
# We could check these first 2 conditions inside unpack_url
# and save repetition of conditions, but then we would
@@ -529,6 +637,7 @@ class RequirementPreparer:
# The file is not available, attempt to fetch only metadata
metadata_dist = self._fetch_metadata_only(req)
if metadata_dist is not None:
_check_linked_requirement_metadata(req, metadata_dist)
req.needs_more_preparation = True
req.set_dist(metadata_dist)
# Ensure download_info is available even in dry-run mode
@@ -586,9 +695,9 @@ class RequirementPreparer:
# We need to verify hashes, and we have found the requirement in the cache
# of locally built wheels.
if (
isinstance(req.download_info.info, ArchiveInfo)
and req.download_info.info.hashes
and hashes.has_one_of(req.download_info.info.hashes)
req.download_info.archive_info
and req.download_info.archive_info.hashes
and hashes.has_one_of(req.download_info.archive_info.hashes)
):
# At this point we know the requirement was built from a hashable source
# artifact, and we verified that the cache entry's hash of the original
@@ -640,14 +749,18 @@ class RequirementPreparer:
# compute it from the downloaded file.
# FIXME: https://github.com/pypa/pip/issues/11943
if (
isinstance(req.download_info.info, ArchiveInfo)
and not req.download_info.info.hashes
req.download_info.archive_info
and not req.download_info.archive_info.hashes
and local_file
):
hash = hash_file(local_file.path)[0].hexdigest()
# We populate info.hash for backward compatibility.
# This will automatically populate info.hashes.
req.download_info.info.hash = f"sha256={hash}"
# We populate archive_info.hashes. For backward compatibility,
# the legacy hash field will be generated when converting to JSON.
req.download_info = DirectUrl(
url=req.download_info.url,
archive_info=ArchiveInfo(hashes={"sha256": hash}),
subdirectory=req.download_info.subdirectory,
)
# For use in later processing,
# preserve the file path on the requirement.
@@ -660,7 +773,30 @@ class RequirementPreparer:
self.build_env_installer,
self.build_isolation,
self.check_build_deps,
self.allow_editables,
)
_check_linked_requirement_metadata(req, dist)
# If a PEP 658 .metadata file was used, check that fields relevant for
# dependency resolution match with the wheel's METADATA file.
#
# NOTE: PEP 658 also permits .metadata files for source distributions,
# but PyPI doesn't serve such files. In addition, an sdist's metadata
# is generated at build time and may legitimately differ from what the
# index declared, so it's been decided to skip this check for sdists.
# This can change later if needed.
#
# TODO: this is a hack for checking whether a distribution is metadata-
# only or not. If/when we refactor distributions to delineate between
# metadata-only and concrete distributions, clean this up.
if (
link.is_wheel
and req._distribution is not None
and req._distribution is not dist
and link.metadata_link() is not None
):
_check_sidecar_matches_wheel(req, req._distribution, dist)
return dist
def save_linked_requirement(self, req: InstallRequirement) -> None:
@@ -683,7 +819,7 @@ class RequirementPreparer:
# No distribution was downloaded for this requirement.
return
download_location = os.path.join(self.download_dir, link.filename)
download_location = join_within_directory(self.download_dir, link.filename)
if not os.path.exists(download_location):
shutil.copy(req.local_file_path, download_location)
download_path = display_path(download_location)
@@ -716,6 +852,7 @@ class RequirementPreparer:
self.build_env_installer,
self.build_isolation,
self.check_build_deps,
self.allow_editables,
)
req.check_if_exists(self.use_user_site)
@@ -44,6 +44,7 @@ def install_given_reqs(
use_user_site: bool,
pycompile: bool,
progress_bar: BarType,
script_executable: str | None = None,
) -> list[InstallationResult]:
"""
Install everything in the given list.
@@ -73,6 +74,7 @@ def install_given_reqs(
for requirement in items:
req_name = requirement.name
assert req_name is not None
if requirement.should_reinstall:
logger.info("Attempting uninstall: %s", req_name)
with indent_log():
@@ -88,6 +90,10 @@ def install_given_reqs(
warn_script_location=warn_script_location,
use_user_site=use_user_site,
pycompile=pycompile,
script_executable=script_executable,
rollback_on_error=bool(
uninstalled_pathset and uninstalled_pathset.can_rollback
),
)
except Exception:
# if install did not succeed, rollback previous uninstall
@@ -14,15 +14,15 @@ import copy
import logging
import os
import re
from collections.abc import Collection
from collections.abc import Collection, Mapping
from dataclasses import dataclass
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging import pylock
from pip._vendor.packaging.markers import InvalidMarker, Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.specifiers import Specifier
from pip._vendor.packaging.utils import parse_sdist_filename, parse_wheel_filename
from pip._internal.exceptions import InstallationError
from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.req.req_file import ParsedRequirement
@@ -30,7 +30,14 @@ from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.filetypes import is_archive_file
from pip._internal.utils.misc import is_installable_dir
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.urls import path_to_url
from pip._internal.utils.pylock import (
package_archive_requirement_url,
package_directory_requirement_url,
package_sdist_requirement_url,
package_vcs_requirement_url,
package_wheel_requirement_url,
)
from pip._internal.utils.urls import path_to_url, strip_extras
from pip._internal.vcs import is_url, vcs
__all__ = [
@@ -40,25 +47,10 @@ __all__ = [
]
logger = logging.getLogger(__name__)
operators = Specifier._operators.keys()
def _strip_extras(path: str) -> tuple[str, str | None]:
m = re.match(r"^(.+)(\[[^\]]+\])$", path)
extras = None
if m:
path_no_extras = m.group(1).rstrip()
extras = m.group(2)
else:
path_no_extras = path
return path_no_extras, extras
def convert_extras(extras: str | None) -> set[str]:
if not extras:
return set()
return get_requirement("placeholder" + extras.lower()).extras
# All standard version specifier operators
# https://packaging.python.org/en/latest/specifications/version-specifiers/#id5
operators = ("~=", "==", "!=", "<=", ">=", "<", ">", "===")
def _set_requirement_extras(req: Requirement, new_extras: set[str]) -> Requirement:
@@ -108,7 +100,7 @@ def _parse_pip_syntax_editable(editable_req: str) -> tuple[str | None, str, set[
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
url_no_extras, extras = strip_extras(url)
if os.path.isdir(url_no_extras):
# Treating it as code that has already been checked out
@@ -116,14 +108,7 @@ def _parse_pip_syntax_editable(editable_req: str) -> tuple[str | None, str, set[
if url_no_extras.lower().startswith("file:"):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
get_requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, set()
return (package_name, url_no_extras, extras)
for version_control in vcs:
if url.lower().startswith(f"{version_control}:"):
@@ -255,7 +240,6 @@ def install_req_from_editable(
hash_options: dict[str, list[str]] | None = None,
constraint: bool = False,
user_supplied: bool = False,
permit_editable_wheels: bool = False,
config_settings: dict[str, str | list[str]] | None = None,
) -> InstallRequirement:
if constraint:
@@ -267,7 +251,6 @@ def install_req_from_editable(
comes_from=comes_from,
user_supplied=user_supplied,
editable=True,
permit_editable_wheels=permit_editable_wheels,
link=parts.link,
constraint=constraint,
isolated=isolated,
@@ -341,19 +324,22 @@ def parse_req_from_line(name: str, line_source: str | None) -> RequirementParts:
if not markers_as_string:
markers = None
else:
markers = Marker(markers_as_string)
try:
markers = Marker(markers_as_string)
except InvalidMarker as exc:
raise InstallationError(f"Invalid requirement: {name.strip()!r}: {exc}")
else:
markers = None
name = name.strip()
req_as_string = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras_as_string = None
if is_url(name):
link = Link(name)
extras: set[str] = set()
else:
p, extras_as_string = _strip_extras(path)
p, extras = strip_extras(path)
url = _get_url_from_path(p, name)
if url is not None:
link = Link(url)
@@ -376,8 +362,6 @@ def parse_req_from_line(name: str, line_source: str | None) -> RequirementParts:
else:
req_as_string = name
extras = convert_extras(extras_as_string)
def with_source(text: str) -> str:
if not line_source:
return text
@@ -453,23 +437,6 @@ def install_req_from_req_string(
except InvalidRequirement as exc:
raise InstallationError(f"Invalid requirement: {req_string!r}: {exc}")
domains_not_allowed = [
PyPI.file_storage_domain,
TestPyPI.file_storage_domain,
]
if (
req.url
and comes_from
and comes_from.link
and comes_from.link.netloc in domains_not_allowed
):
# Explicitly disallow pypi packages that depend on external urls
raise InstallationError(
"Packages installed from PyPI cannot depend on packages "
"which are not also hosted on PyPI.\n"
f"{comes_from.name} depends on {req} "
)
return InstallRequirement(
req,
comes_from,
@@ -546,7 +513,7 @@ def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement:
extras=[],
config_settings=ireq.config_settings,
user_supplied=ireq.user_supplied,
permit_editable_wheels=ireq.permit_editable_wheels,
source_link=ireq.source_link,
)
@@ -566,3 +533,95 @@ def install_req_extend_extras(
else None
)
return result
def _pylock_hashes_to_hash_options(hashes: Mapping[str, str]) -> dict[str, list[str]]:
return {k: [v] for k, v in hashes.items()}
def install_req_from_pylock_package(
package: pylock.Package,
package_dist: (
pylock.PackageVcs
| pylock.PackageArchive
| pylock.PackageDirectory
| pylock.PackageSdist
| pylock.PackageWheel
),
pylock_path_or_url: str,
user_supplied: bool,
) -> tuple[InstallRequirement, Link | None]:
"""Construct an InstallRequirement from a pylock package and artifact.
If the artifact is a Sdist or Wheel, also return a locked Link which
is meant to override candidates from indexes or --find-links.
"""
# TODO: validate file size
if isinstance(package_dist, pylock.PackageVcs):
req_url = package_vcs_requirement_url(pylock_path_or_url, package_dist)
return (
InstallRequirement(
req=Requirement(f"{package.name} @ {req_url}"),
comes_from=pylock_path_or_url,
user_supplied=user_supplied,
),
None,
)
elif isinstance(package_dist, pylock.PackageArchive):
req_url = package_archive_requirement_url(pylock_path_or_url, package_dist)
return (
InstallRequirement(
req=Requirement(f"{package.name} @ {req_url}"),
comes_from=pylock_path_or_url,
hash_options=_pylock_hashes_to_hash_options(package_dist.hashes),
user_supplied=user_supplied,
),
None,
)
elif isinstance(package_dist, pylock.PackageDirectory):
req_url = package_directory_requirement_url(pylock_path_or_url, package_dist)
if package_dist.editable:
return (
install_req_from_editable(
req_url,
comes_from=pylock_path_or_url,
user_supplied=user_supplied,
),
None,
)
else:
return (
install_req_from_line(
req_url,
comes_from=pylock_path_or_url,
user_supplied=user_supplied,
),
None,
)
else:
# wheel or sdist
version = package.version
if isinstance(package_dist, pylock.PackageWheel):
if not version:
_, version, _, _ = parse_wheel_filename(package_dist.filename)
requirement_url = package_wheel_requirement_url(
pylock_path_or_url, package_dist
)
elif isinstance(package_dist, pylock.PackageSdist):
if not version:
_, version = parse_sdist_filename(package_dist.filename)
requirement_url = package_sdist_requirement_url(
pylock_path_or_url, package_dist
)
ireq = InstallRequirement(
req=Requirement(f"{package.name}=={version}"),
comes_from=pylock_path_or_url,
hash_options=_pylock_hashes_to_hash_options(package_dist.hashes),
user_supplied=user_supplied,
)
locked_link = Link(
requirement_url,
comes_from=pylock_path_or_url,
upload_time=package_dist.upload_time,
)
return ireq, locked_link
@@ -14,7 +14,7 @@ class PEP723Exception(ValueError):
def pep723_metadata(scriptfile: str) -> dict[str, Any]:
with open(scriptfile) as f:
with open(scriptfile, encoding="utf8") as f:
script = f.read()
name = "script"
@@ -1,7 +1,8 @@
from collections.abc import Iterable, Iterator
from typing import Any
from pip._vendor.dependency_groups import DependencyGroupResolver
from pip._vendor.packaging.dependency_groups import DependencyGroupResolver
from pip._vendor.packaging.errors import ExceptionGroup
from pip._internal.exceptions import InstallationError
from pip._internal.utils.compat import tomllib
@@ -28,11 +29,13 @@ def _resolve_all_groups(
resolver = resolvers[path]
try:
yield from (str(req) for req in resolver.resolve(groupname))
except (ValueError, TypeError, LookupError) as e:
except ExceptionGroup as eg:
# Convert ExceptionGroup to a single InstallationError with all messages
messages = [str(e) for e in eg.exceptions]
raise InstallationError(
f"[dependency-groups] resolution failed for '{groupname}' "
f"from '{path}': {e}"
) from e
f"from '{path}': {'; '.join(messages)}"
) from eg
def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]:
@@ -54,7 +57,15 @@ def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]:
"Cannot resolve '--group' option."
)
resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
try:
resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
except ExceptionGroup as eg:
# Handle ExceptionGroup from resolver initialization
messages = [str(e) for e in eg.exceptions]
raise InstallationError(
f"[dependency-groups] data was invalid in {path}: {'; '.join(messages)}"
) from eg
return resolvers
@@ -5,7 +5,6 @@ Requirements file parsing
from __future__ import annotations
import codecs
import locale
import logging
import optparse
import os
@@ -13,13 +12,12 @@ import re
import shlex
import sys
import urllib.parse
from collections.abc import Generator, Iterable
from collections.abc import Callable, Generator, Iterable
from dataclasses import dataclass
from optparse import Values
from typing import (
TYPE_CHECKING,
Any,
Callable,
NoReturn,
)
@@ -27,6 +25,7 @@ from pip._internal.cli import cmdoptions
from pip._internal.exceptions import InstallationError, RequirementsFileParseError
from pip._internal.models.release_control import ReleaseControl
from pip._internal.models.search_scope import SearchScope
from pip._internal.utils.compat import get_locale_encoding
if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
@@ -59,6 +58,7 @@ SUPPORTED_OPTIONS: list[Callable[..., optparse.Option]] = [
cmdoptions.only_binary,
cmdoptions.prefer_binary,
cmdoptions.require_hashes,
cmdoptions.no_require_hashes,
cmdoptions.pre,
cmdoptions.all_releases,
cmdoptions.only_final,
@@ -101,18 +101,8 @@ DEFAULT_ENCODING = "utf-8"
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class ParsedRequirement:
# TODO: replace this with slots=True when dropping Python 3.9 support.
__slots__ = (
"requirement",
"is_editable",
"comes_from",
"constraint",
"options",
"line_source",
)
requirement: str
is_editable: bool
comes_from: str
@@ -121,10 +111,8 @@ class ParsedRequirement:
line_source: str | None
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class ParsedLine:
__slots__ = ("filename", "lineno", "args", "opts", "constraint")
filename: str
lineno: int
args: str
@@ -237,6 +225,8 @@ def handle_option_line(
# percolate options upward
if opts.require_hashes:
options.require_hashes = opts.require_hashes
if opts.no_require_hashes:
options.no_require_hashes = opts.no_require_hashes
if opts.features_enabled:
options.features_enabled.extend(
f for f in opts.features_enabled if f not in options.features_enabled
@@ -412,7 +402,7 @@ class RequirementsFileParser:
def _parse_file(
self, filename: str, constraint: bool
) -> Generator[ParsedLine, None, None]:
_, content = get_file_content(filename, self._session)
_, content = get_file_content(filename, self._session, constraint=constraint)
lines_enum = preprocess(content)
@@ -531,8 +521,6 @@ def join_lines(lines_enum: ReqFileLines) -> ReqFileLines:
assert primary_line_number is not None
yield primary_line_number, "".join(new_line)
# TODO: handle space after '\'.
def ignore_comments(lines_enum: ReqFileLines) -> ReqFileLines:
"""
@@ -572,7 +560,9 @@ def expand_env_variables(lines_enum: ReqFileLines) -> ReqFileLines:
yield line_number, line
def get_file_content(url: str, session: PipSession) -> tuple[str, str]:
def get_file_content(
url: str, session: PipSession, *, constraint: bool = False
) -> tuple[str, str]:
"""Gets the content of a file; it may be a filename, file: URL, or
http: URL. Returns (location, content). Content is unicode.
Respects # -*- coding: declarations on the retrieved files.
@@ -595,7 +585,8 @@ def get_file_content(url: str, session: PipSession) -> tuple[str, str]:
with open(url, "rb") as f:
raw_content = f.read()
except OSError as exc:
raise InstallationError(f"Could not open requirements file: {exc}")
kind = "constraint" if constraint else "requirements"
raise InstallationError(f"Could not open {kind} file: {exc}")
content = _decode_req_file(raw_content, url)
@@ -617,7 +608,7 @@ def _decode_req_file(data: bytes, url: str) -> str:
try:
return data.decode(DEFAULT_ENCODING)
except UnicodeDecodeError:
locale_encoding = locale.getpreferredencoding(False) or sys.getdefaultencoding()
locale_encoding = get_locale_encoding() or sys.getdefaultencoding()
logging.warning(
"unable to decode data from %s with default encoding %s, "
"falling back to encoding from locale: %s. "
@@ -34,7 +34,6 @@ from pip._internal.models.direct_url import DirectUrl
from pip._internal.models.link import Link
from pip._internal.operations.build.metadata import generate_metadata
from pip._internal.operations.build.metadata_editable import generate_editable_metadata
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
from pip._internal.req.req_uninstall import UninstallPathSet
from pip._internal.utils.deprecation import deprecated
@@ -49,9 +48,9 @@ from pip._internal.utils.misc import (
redact_auth_from_requirement,
redact_auth_from_url,
)
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.packaging import get_requirement, requirement_without_marker
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryKind
from pip._internal.utils.unpacking import unpack_file
from pip._internal.utils.virtualenv import running_under_virtualenv
from pip._internal.vcs import vcs
@@ -75,19 +74,18 @@ class InstallRequirement:
markers: Marker | None = None,
isolated: bool = False,
*,
source_link: Link | None = None,
hash_options: dict[str, list[str]] | None = None,
config_settings: dict[str, str | list[str]] | None = None,
constraint: bool = False,
extras: Collection[str] = (),
user_supplied: bool = False,
permit_editable_wheels: bool = False,
) -> None:
assert req is None or isinstance(req, Requirement), req
self.req = req
self.comes_from = comes_from
self.constraint = constraint
self.editable = editable
self.permit_editable_wheels = permit_editable_wheels
# source_dir is the local directory where the linked requirement is
# located, or unpacked. In case unpacking is needed, creating and
@@ -107,6 +105,10 @@ class InstallRequirement:
link = Link(req.url)
self.link = self.original_link = link
# The link selected before wheel-cache substitution. Unlike ``link``, this
# does not change when pip substitutes a locally built cached wheel.
self._source_link = source_link if source_link is not None else link
# When this InstallRequirement is a wheel obtained from the cache of locally
# built wheels, this is the source link corresponding to the cache entry, which
# was used to download and build the cached wheel.
@@ -170,6 +172,8 @@ class InstallRequirement:
self.requirements_to_check: list[str] = []
# The PEP 517 backend we should use to build the project
self._pep517_backend_spec: str
self._pep517_backend_path: str | None
self.pep517_backend: BuildBackendHookCaller | None = None
# This requirement needs more preparation before it can be built
@@ -179,8 +183,20 @@ class InstallRequirement:
self._archive_source: Path | None = None
def __str__(self) -> str:
return self._format()
def format_without_marker(self) -> str:
"""Like ``str(self)``, but without the requirement's environment marker.
For error messages, where markers such as ``extra == "foo"`` are noise.
``self.req`` and ``self.markers`` are not modified.
"""
return self._format(include_marker=False)
def _format(self, include_marker: bool = True) -> str:
if self.req:
s = redact_auth_from_requirement(self.req)
req = self.req if include_marker else requirement_without_marker(self.req)
s = redact_auth_from_requirement(req)
if self.link:
s += f" from {redact_auth_from_url(self.link.url)}"
elif self.link:
@@ -229,12 +245,13 @@ class InstallRequirement:
@functools.cached_property
def supports_pyproject_editable(self) -> bool:
assert self.pep517_backend
assert isinstance(self.pep517_backend, ConfiguredBuildBackendHookCaller)
with self.build_env:
runner = runner_with_spinner_message(
"Checking if build backend supports build_editable"
)
with self.pep517_backend.subprocess_runner(runner):
return "build_editable" in self.pep517_backend._supported_features()
return self.pep517_backend.supports_feature("build_editable")
@property
def specifier(self) -> SpecifierSet:
@@ -246,6 +263,19 @@ class InstallRequirement:
"""Whether this requirement was specified as a direct URL."""
return self.original_link is not None
@property
def source_link(self) -> Link | None:
"""The link selected before wheel-cache substitution."""
return self._source_link
def set_source_link(self, link: Link) -> None:
"""Record the selected source link, rejecting later changes."""
if self._source_link is None:
self._source_link = link
return
if self._source_link != link:
raise InstallationError(f"Cannot change the source link for {self}.")
@property
def is_pinned(self) -> bool:
"""Return whether I am pinned to an exact version.
@@ -332,7 +362,7 @@ class InstallRequirement:
# builds (such as numpy). Thus, we ensure that the real path
# is returned.
self._temp_build_dir = TempDirectory(
kind=tempdir_kinds.REQ_BUILD, globally_managed=True
kind=TempDirectoryKind.REQ_BUILD, globally_managed=True
)
return self._temp_build_dir.path
@@ -359,7 +389,7 @@ class InstallRequirement:
return TempDirectory(
path=actual_build_dir,
delete=delete_arg,
kind=tempdir_kinds.REQ_BUILD,
kind=TempDirectoryKind.REQ_BUILD,
globally_managed=True,
).path
@@ -486,11 +516,26 @@ class InstallRequirement:
requires, backend, check, backend_path = pyproject_toml_data
self.requirements_to_check = check
self.pyproject_requires = requires
self._pep517_backend_spec = backend
self._pep517_backend_path = backend_path
def configure_backend(self, python_executable: str) -> None:
"""Set up the build backend hook caller.
This is done separately after pyproject.toml loading as the backend
need to be called with the build environment's Python executable,
which can vary."""
self.pep517_backend = ConfiguredBuildBackendHookCaller(
self,
self.unpacked_source_directory,
backend,
backend_path=backend_path,
self._pep517_backend_spec,
backend_path=self._pep517_backend_path,
python_executable=python_executable,
backend_warning_level=(
logging.WARNING
if self.link and self.link.is_existing_dir()
else logging.DEBUG
),
)
def editable_sanity_check(self) -> None:
@@ -506,7 +551,7 @@ class InstallRequirement:
f"Consider using a build backend that supports PEP 660."
)
def prepare_metadata(self) -> None:
def prepare_metadata(self, allow_editables: bool) -> None:
"""Ensure that project metadata is available.
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
@@ -516,11 +561,7 @@ class InstallRequirement:
details = self.name or f"from {self.link}"
assert self.pep517_backend is not None
if (
self.editable
and self.permit_editable_wheels
and self.supports_pyproject_editable
):
if self.editable and allow_editables and self.supports_pyproject_editable:
self.metadata_directory = generate_editable_metadata(
build_env=self.build_env,
backend=self.pep517_backend,
@@ -764,7 +805,17 @@ class InstallRequirement:
warn_script_location: bool = True,
use_user_site: bool = False,
pycompile: bool = True,
script_executable: str | None = None,
*,
rollback_on_error: bool = False,
) -> None:
# Lazy import to avoid transitively importing `_vendor.distlib.compat`
# which in turn imports `urllib.request` which is slow.
# During an actual installation, `urllib.request` will end up imported anyway,
# but `req.req_install` (this module) is also imported from commands that
# don't actually install anything (e.g. `pip freeze` or `pip show`).
from pip._internal.operations.install.wheel import install_wheel
assert self.req is not None
scheme = get_scheme(
self.req.name,
@@ -787,6 +838,8 @@ class InstallRequirement:
warn_script_location=warn_script_location,
direct_url=self.download_info if self.is_direct else None,
requested=self.user_supplied,
script_executable=script_executable,
rollback_on_error=rollback_on_error,
)
self.install_succeeded = True
@@ -2,19 +2,24 @@ from __future__ import annotations
import functools
import os
import shutil
import sys
import sysconfig
from collections.abc import Generator, Iterable
from collections.abc import Callable, Generator, Iterable
from importlib.util import cache_from_source
from typing import Any, Callable
from typing import Any
from pip._internal.exceptions import LegacyDistutilsInstall, UninstallMissingRecord
from pip._internal.exceptions import (
BlockedImportError,
LegacyDistutilsInstall,
UninstallMissingRecord,
)
from pip._internal.locations import get_bin_prefix, get_bin_user
from pip._internal.metadata import BaseDistribution
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.logging import getLogger, indent_log
from pip._internal.utils.misc import ask, normalize_path, renames, rmtree
from pip._internal.utils.misc import ask, normalize_path, rmtree
from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -96,14 +101,19 @@ def compact(paths: Iterable[str]) -> set[str]:
sep = os.path.sep
short_paths: set[str] = set()
prefixes: set[str] = set()
for path in sorted(paths, key=len):
should_skip = any(
path.startswith(shortpath.rstrip("*"))
and path[len(shortpath.rstrip("*").rstrip(sep))] == sep
for shortpath in short_paths
)
if not should_skip:
current = path[:-1] if path.endswith(sep + "*") else path
parent = current.rstrip(sep)
while parent:
if parent in prefixes:
break
next_parent = os.path.dirname(parent).rstrip(sep)
parent = "" if next_parent == parent else next_parent
else:
short_paths.add(path)
prefixes.add(current.rstrip(sep))
return short_paths
@@ -201,6 +211,8 @@ class StashedUninstallPathSet:
# (old path, new path) tuples for each move that may need
# to be undone.
self._moves: list[tuple[str, str]] = []
# Interrupted transfers must not be retried automatically.
self._manual_recovery: list[tuple[str, str]] = []
def _get_directory_stash(self, path: str) -> str:
"""Stashes a directory.
@@ -244,52 +256,103 @@ class StashedUninstallPathSet:
return save_dir.path
def stash(self, path: str) -> str:
"""Stashes the directory or file and returns its new location.
Handle symlinks as files to avoid modifying the symlink targets.
"""
"""Move a path into temporary storage and return its new location."""
path_is_dir = os.path.isdir(path) and not os.path.islink(path)
if path_is_dir:
new_path = self._get_directory_stash(path)
else:
new_path = self._get_file_stash(path)
self._moves.append((path, new_path))
if path_is_dir and os.path.isdir(new_path):
# If we're moving a directory, we need to
# remove the destination first or else it will be
# moved to inside the existing directory.
# We just created new_path ourselves, so it will
# be removable.
# Remove the empty placeholder so the directory moves to this exact path.
os.rmdir(new_path)
renames(path, new_path)
parent = os.path.dirname(new_path)
if not os.path.exists(parent):
os.makedirs(parent)
try:
shutil.move(path, new_path)
except BaseException:
if os.path.lexists(new_path):
# A failed move may leave either path with only part of the data.
self._manual_recovery.append((path, new_path))
raise
self._moves.append((path, new_path))
old_parent, name = os.path.split(path)
if old_parent and name:
try:
os.removedirs(old_parent)
except OSError:
pass
return new_path
def commit(self) -> None:
"""Commits the uninstall by removing stashed files."""
"""Remove backups unless an interrupted transfer needs manual recovery."""
if self._manual_recovery:
return
for save_dir in self._save_dirs.values():
save_dir.cleanup()
self._moves = []
self._save_dirs = {}
def _restore(self, original: str, stashed: str) -> None:
"""Restore a completed stash and record failures for manual recovery."""
try:
# Check the backup before removing a replacement.
os.lstat(stashed)
if os.path.isfile(original) or os.path.islink(original):
os.unlink(original)
elif os.path.isdir(original):
rmtree(original)
parent, name = os.path.split(original)
if parent and name and not os.path.exists(parent):
os.makedirs(parent)
shutil.move(stashed, original)
except BaseException:
self._manual_recovery.append((original, stashed))
raise
def rollback(self) -> None:
"""Undoes the uninstall by moving stashed files back."""
"""Restore stashed files, keeping backups when restoration fails."""
for p in self._moves:
logger.info("Moving to %s\n from %s", *p)
for new_path, path in self._moves:
try:
logger.debug("Replacing %s from %s", new_path, path)
if os.path.isfile(new_path) or os.path.islink(new_path):
os.unlink(new_path)
elif os.path.isdir(new_path):
rmtree(new_path)
renames(path, new_path)
except OSError as ex:
logger.error("Failed to restore %s", new_path)
logger.debug("Exception: %s", ex)
attempted: set[tuple[str, str]] = set()
try:
for original, stashed in self._moves:
try:
logger.debug("Replacing %s from %s", original, stashed)
attempted.add((original, stashed))
self._restore(original, stashed)
except OSError as ex:
logger.debug("Exception: %s", ex)
finally:
# Retrying an interrupted move could overwrite the complete copy.
self._moves = [move for move in self._moves if move not in attempted]
self.commit()
def report_failed_recovery(self) -> None:
"""Report retained backups that still need manual restoration."""
for original, backup in self.recovery_paths:
logger.error(
"Could not restore %r from %r. Check both paths before retrying.",
original,
backup,
)
@property
def recovery_paths(self) -> tuple[tuple[str, str], ...]:
"""Return original paths and their pending backup locations."""
return tuple(self._moves + self._manual_recovery)
@property
def can_rollback(self) -> bool:
return bool(self._moves)
@@ -369,14 +432,22 @@ class UninstallPathSet:
for_rename = compress_for_rename(self._paths)
for path in sorted(compact(for_rename)):
moved.stash(path)
logger.verbose("Removing file or directory %s", path)
try:
for path in sorted(compact(for_rename)):
moved.stash(path)
logger.verbose("Removing file or directory %s", path)
for pth in self._pth.values():
pth.remove()
for pth in self._pth.values():
pth.remove()
logger.info("Successfully uninstalled %s", dist_name_version)
logger.info("Successfully uninstalled %s", dist_name_version)
except (OSError, BlockedImportError):
if moved.can_rollback:
self.rollback()
else:
moved.commit()
moved.report_failed_recovery()
raise
def _allowed_to_proceed(self, verbose: bool) -> bool:
"""Display which files would be deleted and prompt for confirmation"""
@@ -406,16 +477,24 @@ class UninstallPathSet:
return ask("Proceed (Y/n)? ", ("y", "n", "")) != "n"
@property
def can_rollback(self) -> bool:
return self._moved_paths.can_rollback
def rollback(self) -> None:
"""Rollback the changes previously made by remove()."""
if not self._moved_paths.can_rollback:
# Manual file recovery can still leave saved .pth entries to restore.
if not self.can_rollback and not self._moved_paths.recovery_paths:
logger.error(
"Can't roll back %s; was not uninstalled",
self._dist.raw_name,
)
return
logger.info("Rolling back uninstall of %s", self._dist.raw_name)
self._moved_paths.rollback()
try:
self._moved_paths.rollback()
finally:
self._moved_paths.report_failed_recovery()
for pth in self._pth.values():
pth.rollback()
@@ -609,23 +688,26 @@ class UninstallPthEntries:
if not os.path.isfile(self.file):
logger.warning("Cannot remove entries from nonexistent file %s", self.file)
return
with open(self.file, "rb") as fh:
# windows uses '\r\n' with py3k, but uses '\n' with py2.x
lines = fh.readlines()
self._saved_lines = lines
self._saved_lines = lines.copy()
if any(b"\r\n" in line for line in lines):
endline = "\r\n"
else:
endline = "\n"
# handle missing trailing newline
# Match entries even when the last line has no newline.
if lines and not lines[-1].endswith(endline.encode("utf-8")):
lines[-1] = lines[-1] + endline.encode("utf-8")
for entry in self.entries:
try:
logger.verbose("Removing entry: %s", entry)
lines.remove((entry + endline).encode("utf-8"))
except ValueError:
pass
with open(self.file, "wb") as fh:
fh.writelines(lines)
@@ -633,7 +715,15 @@ class UninstallPthEntries:
if self._saved_lines is None:
logger.error("Cannot roll back changes to %s, none were made", self.file)
return False
logger.debug("Rolling %s back to previous state", self.file)
with open(self.file, "wb") as fh:
fh.writelines(self._saved_lines)
try:
with open(self.file, "wb") as fh:
fh.writelines(self._saved_lines)
except (OSError, BlockedImportError):
logger.error(
"Could not restore %r. Check this file before retrying.", self.file
)
raise
return True
@@ -1,10 +1,10 @@
from typing import Callable, Optional
from collections.abc import Callable
from pip._internal.req.req_install import InstallRequirement
from pip._internal.req.req_set import RequirementSet
InstallRequirementProvider = Callable[
[str, Optional[InstallRequirement]], InstallRequirement
[str, InstallRequirement | None], InstallRequirement
]
@@ -17,7 +17,6 @@ import sys
from collections import defaultdict
from collections.abc import Iterable
from itertools import chain
from typing import Optional
from pip._vendor.packaging import specifiers
from pip._vendor.packaging.requirements import Requirement
@@ -52,7 +51,7 @@ from pip._internal.utils.packaging import check_requires_python
logger = logging.getLogger(__name__)
DiscoveredDependencies = defaultdict[Optional[str], list[InstallRequirement]]
DiscoveredDependencies = defaultdict[str | None, list[InstallRequirement]]
def _check_dist_requires_python(
@@ -124,6 +123,7 @@ class Resolver(BaseResolver):
make_install_req: InstallRequirementProvider,
use_user_site: bool,
ignore_dependencies: bool,
only_dependencies: bool, # unused, for compatibility with new resolver
ignore_installed: bool,
ignore_requires_python: bool,
force_reinstall: bool,
@@ -410,11 +410,13 @@ class Resolver(BaseResolver):
If preparer.require_hashes is True, don't use the wheel cache, because
cached wheels, always built locally, have different hashes than the
files downloaded from the index server and thus throw false hash
mismatches. Furthermore, cached wheels at present have undeterministic
mismatches. Furthermore, cached wheels at present have nondeterministic
contents due to file modification times.
"""
if req.link is None:
req.link = self._find_requirement_link(req)
if req.link is not None:
req.set_source_link(req.link)
if self.wheel_cache is None or self.preparer.require_hashes:
return
@@ -12,7 +12,7 @@ from pip._internal.models.link import Link, links_equivalent
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.hashes import Hashes
CandidateLookup = tuple[Optional["Candidate"], Optional[InstallRequirement]]
CandidateLookup = tuple[Optional["Candidate"], InstallRequirement | None]
def format_name(project: NormalizedName, extras: frozenset[NormalizedName]) -> str:
@@ -26,16 +26,23 @@ def format_name(project: NormalizedName, extras: frozenset[NormalizedName]) -> s
class Constraint:
specifier: SpecifierSet
hashes: Hashes
hash_options: dict[str, list[str]]
links: frozenset[Link]
@classmethod
def empty(cls) -> Constraint:
return Constraint(SpecifierSet(), Hashes(), frozenset())
return Constraint(SpecifierSet(), Hashes(), {}, frozenset())
@classmethod
def from_ireq(cls, ireq: InstallRequirement) -> Constraint:
links = frozenset([ireq.link]) if ireq.link else frozenset()
return Constraint(ireq.specifier, ireq.hashes(trust_internet=False), links)
hash_options = {alg: list(v) for alg, v in ireq.hash_options.items()}
return Constraint(
ireq.specifier,
ireq.hashes(trust_internet=False),
hash_options,
links,
)
def __bool__(self) -> bool:
return bool(self.specifier) or bool(self.hashes) or bool(self.links)
@@ -45,10 +52,19 @@ class Constraint:
return NotImplemented
specifier = self.specifier & other.specifier
hashes = self.hashes & other.hashes(trust_internet=False)
if not self.hash_options:
hash_options = {alg: list(v) for alg, v in other.hash_options.items()}
elif not other.hash_options:
hash_options = {alg: list(v) for alg, v in self.hash_options.items()}
else:
hash_options = {
alg: [v for v in other.hash_options[alg] if v in self.hash_options[alg]]
for alg in self.hash_options.keys() & other.hash_options.keys()
}
links = self.links
if other.link:
links = links.union([other.link])
return Constraint(specifier, hashes, links)
return Constraint(specifier, hashes, hash_options, links)
def is_satisfied_by(self, candidate: Candidate) -> bool:
# Reject if there are any mismatched URL constraints on this package.
@@ -59,6 +75,12 @@ class Constraint:
# prerelease candidates if the user does not expect them.
return self.specifier.contains(candidate.version, prereleases=True)
def format_for_error(self) -> str:
s = str(self.specifier)
if self.links:
s += f" (from {', '.join(str(link) for link in self.links)})"
return s
class Requirement:
@property
@@ -58,10 +58,19 @@ def as_base_candidate(candidate: Candidate) -> BaseCandidate | None:
def make_install_req_from_link(
link: Link, template: InstallRequirement
link: Link,
template: InstallRequirement,
version: Version | None = None,
*,
source_link: Link,
) -> InstallRequirement:
assert not template.editable, "template is editable"
if template.req:
if version is not None and template.req and template.hash_options:
# When hashes are provided via constraints for an unpinned requirement,
# the resulting install requirement must appear pinned so that the
# hash-checking logic does not reject it as HashUnpinned.
line = f"{template.req.name}=={version}"
elif template.req:
line = str(template.req)
else:
line = link.url
@@ -76,6 +85,7 @@ def make_install_req_from_link(
)
ireq.original_link = template.original_link
ireq.link = link
ireq.set_source_link(source_link)
ireq.extras = template.extras
return ireq
@@ -94,7 +104,6 @@ def make_install_req_from_editable(
comes_from=template.comes_from,
isolated=template.isolated,
constraint=template.constraint,
permit_editable_wheels=template.permit_editable_wheels,
hash_options=template.hash_options,
config_settings=template.config_settings,
)
@@ -134,10 +143,6 @@ class _InstallRequirementBackedCandidate(Candidate):
:param link: The link passed to the ``InstallRequirement``. The backing
``InstallRequirement`` will use this link to fetch the distribution.
:param source_link: The link this candidate "originates" from. This is
different from ``link`` when the link is found in the wheel cache.
``link`` would point to the wheel cache, while this points to the
found remote link (e.g. from pypi.org).
"""
dist: BaseDistribution
@@ -146,14 +151,12 @@ class _InstallRequirementBackedCandidate(Candidate):
def __init__(
self,
link: Link,
source_link: Link,
ireq: InstallRequirement,
factory: Factory,
name: NormalizedName | None = None,
version: Version | None = None,
) -> None:
self._link = link
self._source_link = source_link
self._factory = factory
self._ireq = ireq
self._name = name
@@ -181,7 +184,7 @@ class _InstallRequirementBackedCandidate(Candidate):
@property
def source_link(self) -> Link | None:
return self._source_link
return self._ireq.source_link
@property
def project_name(self) -> NormalizedName:
@@ -203,7 +206,8 @@ class _InstallRequirementBackedCandidate(Candidate):
def format_for_error(self) -> str:
return (
f"{self.name} {self.version} "
f"(from {self._link.file_path if self._link.is_file else self._link})"
f"(from {'editable ' if self.is_editable else ''}"
f"{self._link.file_path if self._link.is_file else self._link})"
)
def _prepare_distribution(self) -> BaseDistribution:
@@ -288,7 +292,12 @@ class LinkCandidate(_InstallRequirementBackedCandidate):
if cache_entry is not None:
logger.debug("Using cached wheel link: %s", cache_entry.link)
link = cache_entry.link
ireq = make_install_req_from_link(link, template)
ireq = make_install_req_from_link(
link,
template,
version=version,
source_link=source_link,
)
assert ireq.link == link
if ireq.link.is_wheel and not ireq.link.is_file:
wheel = Wheel(ireq.link.filename)
@@ -317,7 +326,6 @@ class LinkCandidate(_InstallRequirementBackedCandidate):
super().__init__(
link=link,
source_link=source_link,
ireq=ireq,
factory=factory,
name=name,
@@ -342,7 +350,6 @@ class EditableCandidate(_InstallRequirementBackedCandidate):
) -> None:
super().__init__(
link=link,
source_link=link,
ireq=make_install_req_from_editable(link, template),
factory=factory,
name=name,
@@ -366,7 +373,7 @@ class AlreadyInstalledCandidate(Candidate):
self.dist = dist
self._ireq = _make_install_req_from_dist(dist, template)
self._factory = factory
self._version = None
self._version: Version | None = None
# This is just logging some messages, so we can do it eagerly.
# The returned dist would be exactly the same as self.dist because we
@@ -1,12 +1,12 @@
from __future__ import annotations
import contextlib
import copy
import functools
import logging
from collections.abc import Iterable, Iterator, Mapping, Sequence
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from typing import (
TYPE_CHECKING,
Callable,
NamedTuple,
Protocol,
TypeVar,
@@ -18,6 +18,7 @@ from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import InvalidVersion, Version
from pip._vendor.resolvelib import ResolutionImpossible
from pip._vendor.rich.markup import escape
from pip._internal.cache import CacheEntry, WheelCache
from pip._internal.exceptions import (
@@ -45,7 +46,7 @@ from pip._internal.req.req_install import (
from pip._internal.resolution.base import InstallRequirementProvider
from pip._internal.utils.compatibility_tags import get_supported
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.packaging import get_requirement, requirement_without_marker
from pip._internal.utils.virtualenv import running_under_virtualenv
from .base import Candidate, Constraint, Requirement
@@ -212,8 +213,8 @@ class Factory:
except (MetadataInconsistent, MetadataInvalid) as e:
logger.info(
"Discarding [blue underline]%s[/]: [yellow]%s[reset]",
link,
e,
escape(str(link)),
escape(str(e)),
extra={"markup": True},
)
self._build_failures[link] = e
@@ -233,8 +234,8 @@ class Factory:
except MetadataInconsistent as e:
logger.info(
"Discarding [blue underline]%s[/]: [yellow]%s[reset]",
link,
e,
escape(str(link)),
escape(str(e)),
extra={"markup": True},
)
self._build_failures[link] = e
@@ -248,6 +249,7 @@ class Factory:
hashes: Hashes,
prefers_installed: bool,
incompatible_ids: set[int],
constraint_hash_options: dict[str, list[str]] | None = None,
) -> Iterable[Candidate]:
if not ireqs:
return ()
@@ -258,6 +260,16 @@ class Factory:
# Hopefully the Project model can correct this mismatch in the future.
template = ireqs[0]
assert template.req, "Candidates found on index must be PEP 508"
if (
constraint_hash_options
and not template.hash_options
and any(constraint_hash_options.values())
):
template = copy.copy(template)
template.hash_options = {
k: list(v) for k, v in constraint_hash_options.items()
}
assert template.req # to prevent mypy from being confused by the copy
name = canonicalize_name(template.req.name)
extras: frozenset[str] = frozenset()
@@ -376,16 +388,28 @@ class Factory:
This creates "fake" InstallRequirement objects that are basically clones
of what "should" be the template, but with original_link set to link.
"""
extras: frozenset[str] = frozenset()
base_identifier = identifier
with contextlib.suppress(InvalidRequirement):
parsed_requirement = get_requirement(identifier)
if parsed_requirement.name != identifier:
base_identifier = canonicalize_name(parsed_requirement.name)
extras = frozenset(parsed_requirement.extras)
for link in constraint.links:
self._fail_if_link_is_unsupported_wheel(link)
candidate = self._make_base_candidate_from_link(
base_candidate = self._make_base_candidate_from_link(
link,
template=install_req_from_link_and_ireq(link, template),
name=canonicalize_name(identifier),
name=canonicalize_name(base_identifier),
version=None,
)
if candidate:
yield candidate
if base_candidate is None:
continue
if extras:
yield self._make_extras_candidate(base_candidate, extras)
else:
yield base_candidate
def find_candidates(
self,
@@ -453,6 +477,7 @@ class Factory:
constraint.hashes,
prefers_installed,
incompat_ids,
constraint.hash_options,
)
return (
@@ -662,11 +687,12 @@ class Factory:
def _report_single_requirement_conflict(
self, req: Requirement, parent: Candidate | None
) -> DistributionNotFound:
if parent is None:
req_disp = str(req)
_, ireq = req.get_candidate_lookup()
if ireq is not None and ireq.req is not None:
req_str = str(requirement_without_marker(ireq.req))
else:
req_disp = f"{req} (from {parent.name})"
req_str = str(req)
req_disp = req_str if parent is None else f"{req_str} (from {parent.name})"
cands = self._finder.find_all_candidates(req.project_name)
skipped_by_requires_python = self._finder.requires_python_skipped_reasons()
@@ -698,6 +724,7 @@ class Factory:
# Check if only final releases are allowed for this package
version_type = "version"
allows_pre = None
if self._finder.release_control is not None:
allows_pre = self._finder.release_control.allows_prereleases(
canonicalize_name(req.project_name)
@@ -705,13 +732,32 @@ class Factory:
if allows_pre is False:
version_type = "final version"
logger.critical(
"Could not find a %s that satisfies the requirement %s "
"(from versions: %s)",
version_type,
req_disp,
", ".join(versions) or "none",
)
if len(cands) == 1 and cands[0].locked:
# The package finder ensures that requirements from pylock files
# have exactly one candidate. So we can provide a specific error
# message in this case.
if cands[0].version.is_prerelease and allows_pre is False:
logger.critical(
"A pre-release version %s is specified in a provided lock file "
"for %s but only final versions are allowed",
cands[0].version,
cands[0].name,
)
else:
logger.critical(
"The requirement %s is not compatible with "
"version %s specified in a provided lock file",
req_disp,
", ".join(versions),
)
else:
logger.critical(
"Could not find a %s that satisfies the requirement %s "
"(from versions: %s)",
version_type,
req_disp,
", ".join(versions) or "none",
)
if str(req) == "requirements.txt":
logger.info(
"HINT: You are attempting to install a package literally "
@@ -719,8 +765,7 @@ class Factory:
"using the '-r' flag to install the packages listed in "
"requirements.txt"
)
return DistributionNotFound(f"No matching distribution found for {req}")
return DistributionNotFound(f"No matching distribution found for {req_disp}")
def _has_any_candidates(self, project_name: str) -> bool:
"""
@@ -819,8 +864,8 @@ class Factory:
msg = msg + "The user requested "
msg = msg + req.format_for_error()
for key in relevant_constraints:
spec = constraints[key].specifier
msg += f"\n The user requested (constraint) {key}{spec}"
constraint_text = f"{key}{constraints[key].format_for_error()}"
msg += f"\n The user requested (constraint) {constraint_text}"
# Check for causes that had no candidates
causes = set()
@@ -11,8 +11,8 @@ something.
from __future__ import annotations
import logging
from collections.abc import Iterator, Sequence
from typing import Any, Callable, Optional
from collections.abc import Callable, Iterator, Sequence
from typing import Any
from pip._vendor.packaging.version import _BaseVersion
@@ -22,7 +22,7 @@ from .base import Candidate
logger = logging.getLogger(__name__)
IndexCandidateInfo = tuple[_BaseVersion, Callable[[], Optional[Candidate]]]
IndexCandidateInfo = tuple[_BaseVersion, Callable[[], Candidate | None]]
def _iter_built(infos: Iterator[IndexCandidateInfo]) -> Iterator[Candidate]:
@@ -1,6 +1,7 @@
from __future__ import annotations
import math
from collections import defaultdict
from collections.abc import Iterable, Iterator, Mapping, Sequence
from functools import cache
from typing import (
@@ -27,6 +28,8 @@ if TYPE_CHECKING:
else:
_ProviderBase = AbstractProvider
_CONFLICT_PRIORITY_THRESHOLD = 5
# Notes on the relationship between the provider, the factory, and the
# candidate and requirement classes.
#
@@ -81,6 +84,7 @@ class PipProvider(_ProviderBase):
:params constraints: A mapping of constraints specified by the user. Keys
are canonicalized project names.
:params ignore_dependencies: Whether the user specified ``--no-deps``.
:params only_dependencies: Whether the user specified ``--only-deps``.
:params upgrade_strategy: The user-specified upgrade strategy.
:params user_requested: A set of canonicalized package names that the user
supplied for pip to install/upgrade.
@@ -91,14 +95,18 @@ class PipProvider(_ProviderBase):
factory: Factory,
constraints: dict[str, Constraint],
ignore_dependencies: bool,
only_dependencies: bool,
upgrade_strategy: str,
user_requested: dict[str, int],
) -> None:
self._factory = factory
self._constraints = constraints
self._ignore_dependencies = ignore_dependencies
self._only_dependencies = only_dependencies
self._upgrade_strategy = upgrade_strategy
self._user_requested = user_requested
self._conflict_counts: defaultdict[str, int] = defaultdict(int)
self._conflict_promoted: set[str] = set()
@property
def constraints(self) -> dict[str, Constraint]:
@@ -130,29 +138,42 @@ class PipProvider(_ProviderBase):
Further, the current backtrack causes likely need to be resolved
before other requirements as a resolution can't be found while
there is a conflict.
* Identifiers that repeatedly appear as not-yet-pinned in conflicts
get promoted so they are resolved earlier. This lets their
constraints take effect before other packages pick a version.
"""
backtrack_identifiers = set()
for info in backtrack_causes:
backtrack_identifiers.add(info.requirement.name)
names = [info.requirement.name]
if info.parent is not None:
backtrack_identifiers.add(info.parent.name)
names.append(info.parent.name)
for name in names:
backtrack_identifiers.add(name)
if name not in resolutions:
self._conflict_counts[name] += 1
if self._conflict_counts[name] >= _CONFLICT_PRIORITY_THRESHOLD:
self._conflict_promoted.add(name)
current_backtrack_causes = []
promoted = []
for identifier in identifiers:
# Requires-Python has only one candidate and the check is basically
# free, so we always do it first to avoid needless work if it fails.
# This skips calling get_preference() for all other identifiers.
if identifier == REQUIRES_PYTHON_IDENTIFIER:
return [identifier]
# Check if this identifier is a backtrack cause
if identifier in backtrack_identifiers:
current_backtrack_causes.append(identifier)
continue
if identifier in self._conflict_promoted:
promoted.append(identifier)
continue
if current_backtrack_causes:
return current_backtrack_causes
if promoted:
return promoted
return identifiers
def get_preference(
@@ -223,7 +244,10 @@ class PipProvider(_ProviderBase):
unfree = bool(operators)
requested_order = self._user_requested.get(identifier, math.inf)
conflict_promoted = identifier in self._conflict_promoted
return (
not conflict_promoted,
not direct,
not pinned,
not upper_bounded,
@@ -17,8 +17,9 @@ class PipReporter(BaseReporter[Requirement, Candidate, str]):
self.reject_count_by_package: defaultdict[str, int] = defaultdict(int)
self._constraints = constraints or {}
# Only true after a second candidate of this package is rejected.
self._messages_at_reject_count = {
1: (
2: (
"pip is looking at multiple versions of {package_name} to "
"determine which version is compatible with other "
"requirements. This could take a while."
@@ -64,7 +65,7 @@ class PipReporter(BaseReporter[Requirement, Candidate, str]):
name = candidate.name
constraint = self._constraints.get(name)
if constraint and constraint.specifier:
constraint_text = f"{name}{constraint.specifier}"
constraint_text = f"{name}{constraint.format_for_error()}"
msg += f"\n The user requested (constraint) {constraint_text}"
logger.debug(msg)
@@ -49,6 +49,7 @@ class Resolver(BaseResolver):
make_install_req: InstallRequirementProvider,
use_user_site: bool,
ignore_dependencies: bool,
only_dependencies: bool,
ignore_installed: bool,
ignore_requires_python: bool,
force_reinstall: bool,
@@ -57,6 +58,7 @@ class Resolver(BaseResolver):
):
super().__init__()
assert upgrade_strategy in self._allowed_strategies
assert not (ignore_dependencies and only_dependencies)
self.factory = Factory(
finder=finder,
@@ -70,6 +72,7 @@ class Resolver(BaseResolver):
py_version_info=py_version_info,
)
self.ignore_dependencies = ignore_dependencies
self.only_dependencies = only_dependencies
self.upgrade_strategy = upgrade_strategy
self._result: Result | None = None
@@ -81,6 +84,7 @@ class Resolver(BaseResolver):
factory=self.factory,
constraints=collected.constraints,
ignore_dependencies=self.ignore_dependencies,
only_dependencies=self.only_dependencies,
upgrade_strategy=self.upgrade_strategy,
user_requested=collected.user_requested,
)
@@ -181,6 +185,11 @@ class Resolver(BaseResolver):
req_set.add_named_requirement(ireq)
if self.only_dependencies:
for requested in collected.user_requested:
project_name = requested.partition("[")[0]
req_set.requirements.pop(project_name, None)
return req_set
def get_installation_order(
@@ -1,7 +1,6 @@
from __future__ import annotations
import datetime
import functools
import hashlib
import json
import logging
@@ -9,7 +8,6 @@ import optparse
import os.path
import sys
from dataclasses import dataclass
from typing import Callable
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
@@ -17,6 +15,7 @@ from pip._vendor.rich.console import Group
from pip._vendor.rich.markup import escape
from pip._vendor.rich.text import Text
from pip import __version__
from pip._internal.index.collector import LinkCollector
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import get_default_environment
@@ -39,6 +38,7 @@ from pip._internal.utils.misc import (
ExternallyManagedEnvironment,
check_externally_managed,
ensure_dir,
running_under_zipapp,
)
_WEEK = datetime.timedelta(days=7)
@@ -135,35 +135,30 @@ class SelfCheckState:
class UpgradePrompt:
old: str
new: str
show_upgrade_hint: bool = True
def __rich__(self) -> Group:
if WINDOWS:
pip_cmd = f"{get_best_invocation_for_this_python()} -m pip"
else:
pip_cmd = get_best_invocation_for_this_pip()
notice = "[bold][[reset][blue]notice[reset][bold]][reset]"
return Group(
lines = [
Text(),
Text.from_markup(
f"{notice} A new release of pip is available: "
f"[red]{self.old}[reset] -> [green]{self.new}[reset]"
),
Text.from_markup(
f"{notice} To update, run: "
f"[green]{escape(pip_cmd)} install --upgrade pip"
),
)
]
if self.show_upgrade_hint:
if WINDOWS:
pip_cmd = f"{get_best_invocation_for_this_python()} -m pip"
else:
pip_cmd = get_best_invocation_for_this_pip()
def was_installed_by_pip(pkg: str) -> bool:
"""Checks whether pkg was installed by pip
This is used not to display the upgrade message when pip is in fact
installed by system package manager, such as dnf on Fedora.
"""
dist = get_default_environment().get_distribution(pkg)
return dist is not None and "pip" == dist.installer
lines.append(
Text.from_markup(
f"{notice} To update, run: "
f"[green]{escape(pip_cmd)} install --upgrade pip"
)
)
return Group(*lines)
def _get_current_remote_pip_version(
@@ -194,28 +189,18 @@ def _get_current_remote_pip_version(
return str(best_candidate.version)
def _self_version_check_logic(
*,
state: SelfCheckState,
current_time: datetime.datetime,
def _compute_upgrade_prompt(
local_version: Version,
get_remote_version: Callable[[], str | None],
remote_version_str: str,
installed_by_pip: bool,
show_upgrade_hint: bool = True,
) -> UpgradePrompt | None:
remote_version_str = state.get(current_time)
if remote_version_str is None:
remote_version_str = get_remote_version()
if remote_version_str is None:
logger.debug("No remote pip version found")
return None
state.set(remote_version_str, current_time)
remote_version = parse_version(remote_version_str)
logger.debug("Remote version of pip: %s", remote_version)
logger.debug("Local version of pip: %s", local_version)
logger.debug("Was pip installed by pip? %s", installed_by_pip)
pip_installed_by_pip = was_installed_by_pip("pip")
logger.debug("Was pip installed by pip? %s", pip_installed_by_pip)
if not pip_installed_by_pip:
if not installed_by_pip:
return None # Only suggest upgrade if pip is installed by pip.
local_version_is_older = (
@@ -223,33 +208,63 @@ def _self_version_check_logic(
and local_version.base_version != remote_version.base_version
)
if local_version_is_older:
return UpgradePrompt(old=str(local_version), new=remote_version_str)
return UpgradePrompt(
old=str(local_version),
new=remote_version_str,
show_upgrade_hint=show_upgrade_hint,
)
return None
def pip_self_version_check(session: PipSession, options: optparse.Values) -> None:
"""Check for an update for pip.
def pip_self_version_check_fetch(
session: PipSession, options: optparse.Values
) -> UpgradePrompt | None:
"""Compute the pip upgrade prompt, if any, before the command runs.
Limit the frequency of checks to once per week. State is stored either in
the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix
of the pip script path.
"""
installed_dist = get_default_environment().get_distribution("pip")
if not installed_dist:
return
try:
check_externally_managed()
except ExternallyManagedEnvironment:
return
upgrade_prompt = _self_version_check_logic(
state=SelfCheckState(cache_dir=options.cache_dir),
current_time=datetime.datetime.now(datetime.timezone.utc),
local_version=installed_dist.version,
get_remote_version=functools.partial(
_get_current_remote_pip_version, session, options
),
Pair with :func:`pip_self_version_check_emit`, which displays the prompt
after the command body runs.
"""
if running_under_zipapp():
# Use the running pip's version; environment scans target the installed
# pip, not the zipapp. Skip checks that don't apply to zipapps.
local_version = parse_version(__version__)
installed_by_pip = True
else:
installed_dist = get_default_environment().get_distribution("pip")
if not installed_dist:
return None
try:
check_externally_managed()
except ExternallyManagedEnvironment:
return None
local_version = installed_dist.version
installed_by_pip = installed_dist.installer == "pip"
state = SelfCheckState(cache_dir=options.cache_dir)
current_time = datetime.datetime.now(datetime.timezone.utc)
remote_version_str = state.get(current_time)
if remote_version_str is None:
remote_version_str = _get_current_remote_pip_version(session, options)
if remote_version_str is None:
logger.debug("No remote pip version found")
return None
state.set(remote_version_str, current_time)
return _compute_upgrade_prompt(
local_version=local_version,
remote_version_str=remote_version_str,
installed_by_pip=installed_by_pip,
show_upgrade_hint=not running_under_zipapp(),
)
def pip_self_version_check_emit(upgrade_prompt: UpgradePrompt | None) -> None:
"""Emit the upgrade prompt captured by :func:`pip_self_version_check_fetch`."""
if upgrade_prompt is not None:
logger.warning("%s", upgrade_prompt, extra={"rich": True})
@@ -2,12 +2,13 @@
distributions."""
import importlib.resources
import locale
import logging
import os
import sys
from typing import IO
__all__ = ["get_path_uid", "stdlib_pkgs", "tomllib", "WINDOWS"]
__all__ = ["get_locale_encoding", "get_path_uid", "tomllib", "WINDOWS"]
logger = logging.getLogger(__name__)
@@ -26,6 +27,19 @@ def has_tls() -> bool:
return IS_PYOPENSSL
def get_locale_encoding() -> str:
"""Return the locale encoding.
Uses ``locale.getencoding()`` when available (Python 3.11+) to avoid the
``EncodingWarning`` that ``locale.getpreferredencoding(False)`` raises
under UTF-8 Mode.
"""
# TODO: Remove the 3.10 fallback once pip drops Python 3.10 support.
if sys.version_info >= (3, 11):
return locale.getencoding()
return locale.getpreferredencoding(False)
def get_path_uid(path: str) -> int:
"""
Return path's uid.
@@ -73,13 +87,5 @@ else:
from pip._vendor import tomli as tomllib
# packages in the stdlib that may have installation metadata, but should not be
# considered 'installed'. this theoretically could be determined based on
# dist.location (py27:`sysconfig.get_paths()['stdlib']`,
# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may
# make this ineffective, so hard-coding
stdlib_pkgs = {"python", "wsgiref", "argparse"}
# windows detection, covers cpython and ironpython
WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
@@ -5,10 +5,16 @@ A module that implements tooling to enable easy warnings about deprecations.
from __future__ import annotations
import logging
import os
import warnings
from collections.abc import Generator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any, TextIO
from pip._vendor.packaging.version import parse
from pip._vendor.pyproject_hooks import BuildBackendWarning
from pip._vendor.rich.text import Text
from pip import __version__ as current_version # NOTE: tests patch this name.
@@ -16,10 +22,23 @@ DEPRECATION_MSG_PREFIX = "DEPRECATION: "
class PipDeprecationWarning(Warning):
pass
include_source: bool = False
_original_showwarning: Any = None
_build_backend_warning_level = ContextVar(
"build_backend_warning_level", default=logging.DEBUG
)
@contextmanager
def build_backend_warnings(level: int) -> Generator[None, None, None]:
"""Set the logging level for warnings from this build backend call."""
token = _build_backend_warning_level.set(level)
try:
yield
finally:
_build_backend_warning_level.reset(token)
# Warnings <-> Logging Integration
@@ -38,14 +57,30 @@ def _showwarning(
# We use a specially named logger which will handle all of the
# deprecation messages for pip.
logger = logging.getLogger("pip._internal.deprecations")
logger.warning(message)
if isinstance(message, PipDeprecationWarning) and message.include_source:
logger.warning("%s (%s:%s)", message, filename, lineno)
else:
logger.warning(message)
elif issubclass(category, BuildBackendWarning):
logger = logging.getLogger("pip._internal.build_backend")
formatted = warnings.formatwarning(message, category, filename, lineno, line)
logger.log(
_build_backend_warning_level.get(),
"%s",
Text(formatted.rstrip()),
extra={"rich": True},
)
else:
_original_showwarning(message, category, filename, lineno, file, line)
def install_warning_logger() -> None:
# Enable our Deprecation Warnings
warnings.simplefilter("default", PipDeprecationWarning, append=True)
# If we're running pip test suite, promote the PipDeprecationWarning into errors.
if os.environ.get("_PIP_TEST_ENV", None):
warnings.simplefilter("error", PipDeprecationWarning)
else:
warnings.simplefilter("default", PipDeprecationWarning, append=True)
global _original_showwarning
@@ -61,6 +96,8 @@ def deprecated(
gone_in: str | None,
feature_flag: str | None = None,
issue: int | None = None,
stacklevel: int = 2,
include_source: bool = False,
) -> None:
"""Helper to deprecate existing functionality.
@@ -80,6 +117,12 @@ def deprecated(
issue:
Issue number on the tracker that would serve as a useful place for
users to find related discussion and provide feedback.
stacklevel:
How many frames up the call stack to attribute the warning to.
Defaults to 2 (the caller of deprecated()).
include_source:
If True, include the source filename and line number in the warning
output. Useful when the warning originates from external code.
"""
# Determine whether or not the feature is already gone in this version.
@@ -123,4 +166,6 @@ def deprecated(
if is_gone:
raise PipDeprecationWarning(message)
warnings.warn(message, category=PipDeprecationWarning, stacklevel=2)
warning = PipDeprecationWarning(message)
warning.include_source = include_source
warnings.warn(warning, stacklevel=stacklevel)
@@ -11,16 +11,20 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s
direct_url.validate() # if invalid, this is a pip bug
requirement = name + " @ "
fragments = []
if isinstance(direct_url.info, VcsInfo):
if direct_url.vcs_info:
requirement += (
f"{direct_url.info.vcs}+{direct_url.url}@{direct_url.info.commit_id}"
f"{direct_url.vcs_info.vcs}+{direct_url.url}"
f"@{direct_url.vcs_info.commit_id}"
)
elif isinstance(direct_url.info, ArchiveInfo):
elif direct_url.archive_info:
requirement += direct_url.url
if direct_url.info.hash:
fragments.append(direct_url.info.hash)
if direct_url.archive_info.hashes:
hash_algorithm, hash_value = next(
iter(direct_url.archive_info.hashes.items())
)
fragments.append(f"{hash_algorithm}={hash_value}")
else:
assert isinstance(direct_url.info, DirInfo)
assert direct_url.dir_info
requirement += direct_url.url
if direct_url.subdirectory:
fragments.append("subdirectory=" + direct_url.subdirectory)
@@ -32,7 +36,7 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s
def direct_url_for_editable(source_dir: str) -> DirectUrl:
return DirectUrl(
url=path_to_url(source_dir),
info=DirInfo(editable=True),
dir_info=DirInfo(editable=True),
)
@@ -62,7 +66,7 @@ def direct_url_from_link(
commit_id = vcs_backend.get_revision(source_dir)
return DirectUrl(
url=url,
info=VcsInfo(
vcs_info=VcsInfo(
vcs=vcs_backend.name,
commit_id=commit_id,
requested_revision=requested_revision,
@@ -72,16 +76,17 @@ def direct_url_from_link(
elif link.is_existing_dir():
return DirectUrl(
url=link.url_without_fragment,
info=DirInfo(),
dir_info=DirInfo(),
subdirectory=link.subdirectory_fragment,
)
else:
hash = None
hash_name = link.hash_name
if hash_name:
hash = f"{hash_name}={link.hash}"
if link.hash_name:
assert link.hash
hashes = {link.hash_name: link.hash}
else:
hashes = None
return DirectUrl(
url=link.url_without_fragment,
info=ArchiveInfo(hash=hash),
archive_info=ArchiveInfo(hashes=hashes),
subdirectory=link.subdirectory_fragment,
)
@@ -5,11 +5,11 @@ import os
import os.path
import random
import sys
from collections.abc import Generator
from collections.abc import Callable, Generator
from contextlib import contextmanager
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, BinaryIO, Callable, cast
from typing import Any, BinaryIO, cast
from pip._internal.utils.compat import get_path_uid
from pip._internal.utils.misc import format_size
@@ -172,7 +172,7 @@ def _subdirs_without_generic(
predicate under it."""
directories = []
excluded = set()
excluded: set[Path] = set()
for root_str, _, filenames in os.walk(Path(path).resolve()):
root = Path(root_str)
@@ -180,9 +180,7 @@ def _subdirs_without_generic(
# This directory should be excluded, so exclude it and all of its
# parent directories.
# The last item in root.parents is ".", so we ignore it.
#
# Wrapping this in `list()` is only needed for Python 3.9.
excluded.update(list(root.parents)[:-1])
excluded.update(root.parents[:-1])
excluded.add(root)
directories.append(root)
@@ -8,9 +8,9 @@ from pip._internal.exceptions import HashMismatch, HashMissing, InstallationErro
from pip._internal.utils.misc import read_chunks
if TYPE_CHECKING:
from collections.abc import Mapping
from hashlib import _Hash
# The recommended hash algo of the moment. Change this whenever the state of
# the art changes; it won't hurt backward compatibility.
FAVORITE_HASH = "sha256"
@@ -104,7 +104,7 @@ class Hashes:
with open(path, "rb") as file:
return self.check_against_file(file)
def has_one_of(self, hashes: dict[str, str]) -> bool:
def has_one_of(self, hashes: Mapping[str, str]) -> bool:
"""Return whether any of the given hashes are allowed."""
for hash_name, hex_digest in hashes.items():
if self.is_hash_allowed(hash_name, hex_digest):
@@ -29,7 +29,7 @@ from pip._vendor.rich.style import Style
from pip._internal.utils._log import VERBOSE, getLogger
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
from pip._internal.utils.misc import StreamWrapper, ensure_dir
from pip._internal.utils.misc import StreamWrapper, ensure_dir, looks_like_ci
_log_state = threading.local()
_stdout_console = None
@@ -320,8 +320,16 @@ def setup_logging(verbosity: int, no_color: bool, user_log_file: str | None) ->
["user_log"] if include_user_log else []
)
global _stdout_console, stderr_console
_stdout_console = PipConsole(file=sys.stdout, no_color=no_color, soft_wrap=True)
_stderr_console = PipConsole(file=sys.stderr, no_color=no_color, soft_wrap=True)
console_options: dict[str, Any] = {"no_color": no_color, "soft_wrap": True}
if looks_like_ci():
# Don't animate progress bars and status spinners when running
# in CI. A lot of CI workflows use FORCE_COLOR and similar envvars
# so their CI output remains colorized, but this causes rich to
# assume a fully interactive terminal, resulting in any animated
# output to span many lines which is distracting (and unhelpful).
console_options["force_interactive"] = False
_stdout_console = PipConsole(file=sys.stdout, **console_options)
_stderr_console = PipConsole(file=sys.stderr, **console_options)
logging.config.dictConfig(
{
@@ -5,15 +5,16 @@ import getpass
import hashlib
import logging
import os
import pathlib
import posixpath
import shutil
import stat
import sys
import sysconfig
import urllib.parse
from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from functools import cache, partial, wraps
from io import StringIO
from itertools import filterfalse, tee, zip_longest
from pathlib import Path
@@ -21,20 +22,25 @@ from types import FunctionType, TracebackType
from typing import (
Any,
BinaryIO,
Callable,
Optional,
NoReturn,
TextIO,
TypeVar,
cast,
)
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pip._vendor.pyproject_hooks import BackendUnavailable, BuildBackendHookCaller
from pip import __file__ as pip_location
from pip import __version__
from pip._internal.exceptions import CommandError, ExternallyManagedEnvironment
from pip._internal.exceptions import (
BackendUnavailableError,
CommandError,
ExternallyManagedEnvironment,
)
from pip._internal.locations import get_major_minor_version
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.deprecation import build_backend_warnings
from pip._internal.utils.retry import retry
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -52,19 +58,38 @@ __all__ = [
"ensure_dir",
"remove_auth_from_url",
"check_externally_managed",
"looks_like_ci",
"ConfiguredBuildBackendHookCaller",
]
logger = logging.getLogger(__name__)
T = TypeVar("T")
F = TypeVar("F", bound=Callable[..., Any])
ExcInfo = tuple[type[BaseException], BaseException, TracebackType]
VersionInfo = tuple[int, int, int]
NetlocTuple = tuple[str, tuple[Optional[str], Optional[str]]]
NetlocTuple = tuple[str, tuple[str | None, str | None]]
OnExc = Callable[[FunctionType, Path, BaseException], Any]
OnErr = Callable[[FunctionType, Path, ExcInfo], Any]
FILE_CHUNK_SIZE = 1024 * 1024
# These are environment variables present when running under various
# CI systems. For each variable, some CI systems that use the variable
# are indicated. The collection was chosen so that for each of a number
# of popular systems, at least one of the environment variables is used.
# This list is used to provide some indication of and lower bound for
# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive.
# For more background, see: https://github.com/pypa/pip/issues/5499
CI_ENVIRONMENT_VARIABLES = (
# Azure Pipelines
"BUILD_BUILDID",
# Jenkins
"BUILD_ID",
# AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI
"CI",
# Explicit environment variable.
"PIP_IS_CI",
)
def get_pip_version() -> str:
@@ -74,6 +99,27 @@ def get_pip_version() -> str:
return f"pip {__version__} from {pip_pkg_dir} (python {get_major_minor_version()})"
@cache
def running_under_zipapp() -> bool:
return not pathlib.Path(pip_location).resolve().parent.is_dir()
def get_runnable_pip() -> str:
"""Get a file to pass to a Python executable, to run the currently-running pip.
This is used to run a pip subprocess, for installing requirements into the build
environment.
"""
source = pathlib.Path(pip_location).resolve().parent
if running_under_zipapp():
# This would happen if someone is using pip from inside a zip file. In that
# case, we can use that directly.
return str(source)
return os.fsdecode(source / "__pip-runner__.py")
def normalize_version_info(py_version_info: tuple[int, ...]) -> tuple[int, int, int]:
"""
Convert a tuple of ints representing a Python version to one of length
@@ -385,14 +431,6 @@ class StreamWrapper(StringIO):
return self.orig_stream.encoding
# Simulates an enum
def enum(*sequential: Any, **named: Any) -> type[Any]:
enums = dict(zip(sequential, range(len(sequential))), **named)
reverse = {value: key for key, value in enums.items()}
enums["reverse_mapping"] = reverse
return type("Enum", (), enums)
def build_netloc(host: str, port: int | None) -> str:
"""
Build a netloc from a host-port pair
@@ -548,7 +586,7 @@ class HiddenText:
if type(self) is type(other):
# The string being used for redaction doesn't also have to match,
# just the raw, original string.
return self.secret == cast(HiddenText, other).secret
return self.secret == other.secret
return NotImplemented
# Disable hashing, since we have a custom __eq__ and don't need hash-ability
@@ -648,6 +686,24 @@ def partition(
return filterfalse(pred, t1), filter(pred, t2)
def _build_backend_hook(func: F) -> F:
"""Apply warning policy and translate failures for a public backend hook."""
@wraps(func)
def wrapper(
self: ConfiguredBuildBackendHookCaller,
*args: Any,
**kwargs: Any,
) -> Any:
with build_backend_warnings(self.backend_warning_level):
try:
return func(self, *args, **kwargs)
except BackendUnavailable as error:
self._raise_backend_unavailable(func.__name__, error)
return cast(F, wrapper)
class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
def __init__(
self,
@@ -657,12 +713,34 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
backend_path: str | None = None,
runner: Callable[..., None] | None = None,
python_executable: str | None = None,
*,
backend_warning_level: int = logging.DEBUG,
):
super().__init__(
source_dir, build_backend, backend_path, runner, python_executable
)
self.config_holder = config_holder
self.backend_warning_level = backend_warning_level
def _raise_backend_unavailable(
self, hook_name: str, error: BackendUnavailable
) -> NoReturn:
backend_error = (
error.traceback.strip().splitlines()[-1] if error.traceback else str(error)
)
exception = BackendUnavailableError(
hook_name=hook_name,
backend_name=self.build_backend,
backend_error=backend_error,
)
logger.error("%s", exception, extra={"rich": True})
raise exception from error
@_build_backend_hook
def supports_feature(self, feature: str) -> bool:
return feature in self._supported_features()
@_build_backend_hook
def build_wheel(
self,
wheel_directory: str,
@@ -674,6 +752,7 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
wheel_directory, config_settings=cs, metadata_directory=metadata_directory
)
@_build_backend_hook
def build_sdist(
self,
sdist_directory: str,
@@ -682,6 +761,7 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
cs = self.config_holder.config_settings
return super().build_sdist(sdist_directory, config_settings=cs)
@_build_backend_hook
def build_editable(
self,
wheel_directory: str,
@@ -693,24 +773,28 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
wheel_directory, config_settings=cs, metadata_directory=metadata_directory
)
@_build_backend_hook
def get_requires_for_build_wheel(
self, config_settings: Mapping[str, Any] | None = None
) -> Sequence[str]:
cs = self.config_holder.config_settings
return super().get_requires_for_build_wheel(config_settings=cs)
@_build_backend_hook
def get_requires_for_build_sdist(
self, config_settings: Mapping[str, Any] | None = None
) -> Sequence[str]:
cs = self.config_holder.config_settings
return super().get_requires_for_build_sdist(config_settings=cs)
@_build_backend_hook
def get_requires_for_build_editable(
self, config_settings: Mapping[str, Any] | None = None
) -> Sequence[str]:
cs = self.config_holder.config_settings
return super().get_requires_for_build_editable(config_settings=cs)
@_build_backend_hook
def prepare_metadata_for_build_wheel(
self,
metadata_directory: str,
@@ -724,12 +808,13 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
_allow_fallback=_allow_fallback,
)
@_build_backend_hook
def prepare_metadata_for_build_editable(
self,
metadata_directory: str,
config_settings: Mapping[str, Any] | None = None,
_allow_fallback: bool = True,
) -> str | None:
) -> str:
cs = self.config_holder.config_settings
return super().prepare_metadata_for_build_editable(
metadata_directory=metadata_directory,
@@ -769,3 +854,13 @@ def warn_if_run_as_root() -> None:
"Use the --root-user-action option if you know what you are doing and "
"want to suppress this warning."
)
def looks_like_ci() -> bool:
"""
Return whether it looks like pip is running under CI.
"""
# We don't use the method of checking for a tty (e.g. using isatty())
# because some CI systems mimic a tty (e.g. Travis CI). Thus that
# method doesn't provide definitive information in either direction.
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
@@ -1,5 +1,6 @@
from __future__ import annotations
import copy
import functools
import logging
@@ -42,3 +43,14 @@ def get_requirement(req_string: str) -> Requirement:
# minimize repeated parsing of the same string to construct equivalent
# Requirement objects.
return Requirement(req_string)
def requirement_without_marker(req: Requirement) -> Requirement:
"""Return a copy of ``req`` with its environment marker removed.
Intended for error messages. The original is left unchanged, which
matters because ``get_requirement`` caches and shares instances.
"""
stripped = copy.copy(req)
stripped.marker = None
return stripped
@@ -1,5 +1,10 @@
from collections.abc import Iterable
from __future__ import annotations
import os
from collections.abc import Collection, Iterable, Iterator
from pathlib import Path
from typing import TYPE_CHECKING
from urllib.parse import urljoin, urlsplit
from pip._vendor.packaging.pylock import (
Package,
@@ -9,13 +14,20 @@ from pip._vendor.packaging.pylock import (
PackageVcs,
PackageWheel,
Pylock,
is_valid_pylock_path,
)
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo
from pip._internal.exceptions import DiagnosticPipError, InstallationError
from pip._internal.models.link import Link
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.urls import url_to_path
from pip._internal.utils.compat import tomllib
from pip._internal.utils.urls import path_to_url, url_to_path
if TYPE_CHECKING:
from pip._internal.index.package_finder import FormatControl
from pip._internal.network.session import PipSession
from pip._internal.req.req_install import InstallRequirement
def _pylock_package_from_install_requirement(
@@ -32,16 +44,16 @@ def _pylock_package_from_install_requirement(
package_sdist = None
package_wheels = None
if ireq.is_direct:
if isinstance(download_info.info, VcsInfo):
if download_info.vcs_info:
package_vcs = PackageVcs(
type=download_info.info.vcs,
type=download_info.vcs_info.vcs,
url=download_info.url,
path=None,
requested_revision=download_info.info.requested_revision,
commit_id=download_info.info.commit_id,
requested_revision=download_info.vcs_info.requested_revision,
commit_id=download_info.vcs_info.commit_id,
subdirectory=download_info.subdirectory,
)
elif isinstance(download_info.info, DirInfo):
elif download_info.dir_info:
package_directory = PackageDirectory(
path=(
Path(url_to_path(download_info.url))
@@ -50,17 +62,19 @@ def _pylock_package_from_install_requirement(
.as_posix()
),
editable=(
download_info.info.editable if download_info.info.editable else None
download_info.dir_info.editable
if download_info.dir_info.editable
else None
),
subdirectory=download_info.subdirectory,
)
elif isinstance(download_info.info, ArchiveInfo):
if not download_info.info.hashes:
elif download_info.archive_info:
if not download_info.archive_info.hashes:
raise NotImplementedError()
package_archive = PackageArchive(
url=download_info.url,
path=None,
hashes=download_info.info.hashes,
hashes=download_info.archive_info.hashes,
subdirectory=download_info.subdirectory,
)
else:
@@ -68,8 +82,8 @@ def _pylock_package_from_install_requirement(
raise NotImplementedError()
else:
package_version = dist.version
if isinstance(download_info.info, ArchiveInfo):
if not download_info.info.hashes:
if download_info.archive_info:
if not download_info.archive_info.hashes:
raise NotImplementedError()
link = Link(download_info.url)
if link.is_wheel:
@@ -77,14 +91,14 @@ def _pylock_package_from_install_requirement(
PackageWheel(
name=link.filename,
url=download_info.url,
hashes=download_info.info.hashes,
hashes=download_info.archive_info.hashes,
)
]
else:
package_sdist = PackageSdist(
name=link.filename,
url=download_info.url,
hashes=download_info.info.hashes,
hashes=download_info.archive_info.hashes,
)
else:
# should never happen
@@ -114,3 +128,197 @@ def pylock_from_install_requirements(
key=lambda p: p.name,
),
)
def _is_url(s: str) -> bool:
return s.lower().startswith(("http:", "https:", "file:"))
def is_valid_pylock_filename(filename: str, *, allow_urls: bool = True) -> bool:
"""Check a lock source's filename, optionally restricting it to local paths."""
if _is_url(filename):
if not allow_urls:
return False
try:
path = Path(urlsplit(filename).path.rpartition("/")[-1])
except ValueError:
return False
else:
path = Path(filename)
return is_valid_pylock_path(path)
def _package_dist_url(
pylock_path_or_url: str, path: str | None, url: str | None
) -> str:
"""Compute an url from a Pylock package path and url.
Give priority to path over url. If path is relative,
compute an url using the pylock file location as base.
"""
if path is not None:
if not os.path.isabs(path):
# relative path, join to pylock location
if _is_url(pylock_path_or_url):
base = urlsplit(pylock_path_or_url)
dist_url = urljoin(base.geturl(), path)
# os.path.isabs does not treat a scheme-carrying value like
# "file:..." as absolute, so it reaches here and urljoin honors
# its scheme, discarding the pylock base. Only keep the result
# if its scheme and host still match the lock's own.
target = urlsplit(dist_url)
if (target.scheme, target.netloc) != (base.scheme, base.netloc):
raise InstallationError(
f"Path {path!r} in pylock file obtained from a URL "
f"resolves outside its location: {pylock_path_or_url!r}"
)
return dist_url
else:
return path_to_url(
os.path.join(os.path.dirname(pylock_path_or_url), path)
)
else:
# absolute path, reject if pylock comes from a URL
if _is_url(pylock_path_or_url):
raise InstallationError(
f"Absolute paths are not supported in pylock files obtained "
f"from a URL: {path!r} in {pylock_path_or_url!r}"
)
return path_to_url(path)
else:
assert url is not None # guaranteed by packaging.pylock validation
return url
def package_vcs_requirement_url(
pylock_path_or_url: str, package_vcs: PackageVcs
) -> str:
dist_url = _package_dist_url(pylock_path_or_url, package_vcs.path, package_vcs.url)
url = f"{package_vcs.type}+{dist_url}@{package_vcs.commit_id}"
if package_vcs.subdirectory:
if "#" in url:
raise InstallationError(
f"Package URL {url!r} cannot contain fragments in combination "
f"with subdirectory field (in {pylock_path_or_url!r})"
)
url += "#subdirectory=" + package_vcs.subdirectory
return url
def package_archive_requirement_url(
pylock_path_or_url: str, package_archive: PackageArchive
) -> str:
url = _package_dist_url(
pylock_path_or_url, package_archive.path, package_archive.url
)
if package_archive.subdirectory:
if "#" in url:
raise InstallationError(
f"Package URL {url!r} cannot contain fragments in combination "
f"with subdirectory field (in {pylock_path_or_url!r})"
)
url += "#subdirectory=" + package_archive.subdirectory
return url
def package_directory_requirement_url(
pylock_path_or_url: str, package_directory: PackageDirectory
) -> str:
if _is_url(pylock_path_or_url) and not pylock_path_or_url.lower().startswith(
"file:"
):
raise InstallationError(
f"Directory entries are not supported in remote pylock.toml "
f"{pylock_path_or_url!r}"
)
url = _package_dist_url(pylock_path_or_url, package_directory.path, None)
assert url.startswith("file://")
if not url.endswith("/"):
url += "/"
if package_directory.subdirectory:
url += package_directory.subdirectory
if not url.endswith("/"):
url += "/"
return url
def package_sdist_requirement_url(
pylock_path_or_url: str, package_sdist: PackageSdist
) -> str:
return _package_dist_url(pylock_path_or_url, package_sdist.path, package_sdist.url)
def package_wheel_requirement_url(
pylock_path_or_url: str, package_wheel: PackageWheel
) -> str:
return _package_dist_url(pylock_path_or_url, package_wheel.path, package_wheel.url)
def _get_pylock_path_or_url_content(path_or_url: str, session: PipSession) -> str:
# TODO: refactor - this is similar to req_file.get_file_content
# Pip has special support for file:// URLs (LocalFSAdapter).
if _is_url(path_or_url):
# Delay importing heavy network modules until absolutely necessary.
from pip._internal.network.utils import raise_for_status
resp = session.get(path_or_url)
raise_for_status(resp)
return resp.text
# Assume this is a bare path.
return Path(path_or_url).read_text(encoding="utf-8")
def select_from_pylock_path_or_url(
pylock_path_or_url: str,
session: PipSession,
format_control: FormatControl | None,
*,
extras: Collection[str] | None = None,
dependency_groups: Iterable[str] = (),
include_default_groups: bool = True,
) -> Iterator[
tuple[
Package,
PackageVcs | PackageDirectory | PackageArchive | PackageWheel | PackageSdist,
]
]:
try:
pylock_content = _get_pylock_path_or_url_content(pylock_path_or_url, session)
except DiagnosticPipError:
raise
except Exception as exc:
raise InstallationError(
f"Error reading pylock file {pylock_path_or_url!r}: {exc}"
) from exc
try:
lock = Pylock.from_dict(tomllib.loads(pylock_content))
except Exception as exc:
raise InstallationError(
f"Invalid pylock file {pylock_path_or_url!r}: {exc}"
) from exc
selected_groups: set[str] = {canonicalize_name(name) for name in dependency_groups}
if include_default_groups:
selected_groups.update(lock.default_groups or ())
def prefer_sdist_predicate(name: NormalizedName) -> bool:
if format_control is None:
return False
allowed_formats = format_control.get_allowed_formats(name)
if "source" in allowed_formats and "binary" not in allowed_formats:
return True
return False
try:
# packaging 26.4 will reject undeclared extras and dependency groups.
yield from lock.select(
extras=extras,
dependency_groups=selected_groups,
prefer_sdist_predicate=prefer_sdist_predicate,
)
except Exception as exc:
raise InstallationError(
f"Cannot select requirements from pylock file {pylock_path_or_url!r}: {exc}"
) from exc
@@ -1,8 +1,9 @@
from __future__ import annotations
import functools
from collections.abc import Callable
from time import perf_counter, sleep
from typing import TYPE_CHECKING, Callable, TypeVar
from typing import TYPE_CHECKING, TypeVar
if TYPE_CHECKING:
from typing_extensions import ParamSpec
@@ -4,8 +4,8 @@ import logging
import os
import shlex
import subprocess
from collections.abc import Iterable, Mapping
from typing import Any, Callable, Literal, Union
from collections.abc import Callable, Iterable, Mapping
from typing import Any, Literal
from pip._vendor.rich.markup import escape
@@ -14,7 +14,7 @@ from pip._internal.exceptions import InstallationSubprocessError
from pip._internal.utils.logging import VERBOSE, subprocess_logger
from pip._internal.utils.misc import HiddenText
CommandArgs = list[Union[str, HiddenText]]
CommandArgs = list[str | HiddenText]
def make_command(*args: str | HiddenText | CommandArgs) -> CommandArgs:
@@ -134,6 +134,7 @@ def call_subprocess(
stderr=subprocess.STDOUT if not stdout_only else subprocess.PIPE,
cwd=cwd,
env=env,
encoding="locale",
errors="backslashreplace",
)
except Exception as exc:
@@ -6,16 +6,16 @@ import logging
import os.path
import tempfile
import traceback
from collections.abc import Generator
from collections.abc import Callable, Generator
from contextlib import ExitStack, contextmanager
from enum import Enum
from pathlib import Path
from typing import (
Any,
Callable,
TypeVar,
)
from pip._internal.utils.misc import enum, rmtree
from pip._internal.utils.misc import rmtree
logger = logging.getLogger(__name__)
@@ -24,11 +24,13 @@ _T = TypeVar("_T", bound="TempDirectory")
# Kinds of temporary directories. Only needed for ones that are
# globally-managed.
tempdir_kinds = enum(
BUILD_ENV="build-env",
EPHEM_WHEEL_CACHE="ephem-wheel-cache",
REQ_BUILD="req-build",
)
# TODO: Use StrEnum when Python 3.10 support is dropped.
class TempDirectoryKind(str, Enum):
BUILD_ENV = "build-env"
EPHEM_WHEEL_CACHE = "ephem-wheel-cache"
REQ_BUILD = "req-build"
__str__ = str.__str__
_tempdir_manager: ExitStack | None = None
@@ -76,15 +76,26 @@ def has_leading_dir(paths: Iterable[str]) -> bool:
return True
def is_within_directory(directory: str, target: str) -> bool:
def is_within_directory(
directory: str, target: str, *, resolve_symlinks: bool = False
) -> bool:
"""
Return true if the absolute path of target is within the directory
"""
abs_directory = os.path.abspath(directory)
abs_target = os.path.abspath(target)
(including when target is equal to the directory).
prefix = os.path.commonpath([abs_directory, abs_target])
return prefix == abs_directory
When ``resolve_symlinks`` is true, resolve symlinks before comparing so
traversal through a symlink (e.g. "link/../file") is also caught.
"""
if resolve_symlinks:
abs_directory = os.path.realpath(directory)
abs_target = os.path.realpath(target)
else:
abs_directory = os.path.abspath(directory)
abs_target = os.path.abspath(target)
return abs_target == abs_directory or abs_target.startswith(
os.path.join(abs_directory, "")
)
def _get_default_mode_plus_executable() -> int:
@@ -277,7 +288,13 @@ def _untar_without_filter(
if leading:
fn = split_leading_dir(fn)[1]
path = os.path.join(location, fn)
if not is_within_directory(location, path):
# The plain check rejects textual ".." escapes; resolving symlinks also
# catches a later member redirected outside by an earlier member's
# symlink (e.g. "link/../file").
if not is_within_directory(location, path) or not is_within_directory(
location, path, resolve_symlinks=True
):
message = (
"The tar file ({}) has a file ({}) trying to install "
"outside target directory ({})"
@@ -286,6 +303,17 @@ def _untar_without_filter(
if member.isdir():
ensure_dir(path)
elif member.issym():
# Reject symlinks resolving outside the destination, so a later
# member cannot be written through them.
target = os.path.join(os.path.dirname(path), member.linkname)
if not is_within_directory(location, target, resolve_symlinks=True):
message = (
"The tar file ({}) has a file ({}) trying to install "
"outside target directory ({})"
)
raise InstallationError(
message.format(filename, member.name, member.linkname)
)
if not is_symlink_target_in_tar(tar, member):
message = (
"The tar file ({}) has a file ({}) trying to install "
@@ -336,27 +364,46 @@ def unpack_file(
location: str,
content_type: str | None = None,
) -> None:
"""Unpack ``filename`` into ``location``.
Archive format is chosen in order of decreasing reliability:
``content_type``, then filename extension, then magic signature
(unambiguous matches only).
"""
filename = os.path.realpath(filename)
if (
content_type == "application/zip"
or filename.lower().endswith(ZIP_EXTENSIONS)
or zipfile.is_zipfile(filename)
):
unzip_file(filename, location, flatten=not filename.endswith(".whl"))
elif (
content_type == "application/x-gzip"
or tarfile.is_tarfile(filename)
or filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)
):
zip_flatten = not filename.endswith(".whl")
def _unzip() -> None:
unzip_file(filename, location, flatten=zip_flatten)
def _untar() -> None:
untar_file(filename, location)
else:
# FIXME: handle?
# FIXME: magic signatures?
logger.critical(
"Cannot unpack file %s (downloaded from %s, content-type: %s); "
"cannot detect archive format",
filename,
location,
content_type,
)
raise InstallationError(f"Cannot determine archive format of {location}")
if content_type == "application/zip":
return _unzip()
if content_type == "application/x-gzip":
return _untar()
if filename.lower().endswith(ZIP_EXTENSIONS):
return _unzip()
if filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS):
return _untar()
# avoid ambiguous case where both signature checks return True
is_zipfile = zipfile.is_zipfile(filename)
is_tarfile = tarfile.is_tarfile(filename)
if is_zipfile and not is_tarfile:
return _unzip()
if is_tarfile and not is_zipfile:
return _untar()
if is_zipfile and is_tarfile:
logger.error("Ambiguous file signature in %s.", filename)
logger.critical(
"Cannot unpack file %s (downloaded from %s, content-type: %s); "
"cannot detect archive format",
filename,
location,
content_type,
)
raise InstallationError(f"Cannot determine archive format of {location}")
@@ -1,9 +1,10 @@
import os
import re
import string
import urllib.parse
import urllib.request
import urllib.parse # noqa: F401
from .compat import WINDOWS
from .packaging import get_requirement
def path_to_url(path: str) -> str:
@@ -11,6 +12,8 @@ def path_to_url(path: str) -> str:
Convert a path to a file: URL. The path will be made absolute and have
quoted path parts.
"""
import urllib.request
path = os.path.normpath(os.path.abspath(path))
url = urllib.parse.urljoin("file://", urllib.request.pathname2url(path))
return url
@@ -20,6 +23,8 @@ def url_to_path(url: str) -> str:
"""
Convert a file: URL to a path.
"""
import urllib.request
assert url.startswith(
"file:"
), f"You can only turn file: urls into filenames (not {url!r})"
@@ -53,3 +58,21 @@ def url_to_path(url: str) -> str:
path = path[1:]
return path
def _convert_extras(extras: str | None) -> set[str]:
if not extras:
return set()
return get_requirement("placeholder" + extras.lower()).extras
def strip_extras(path: str) -> tuple[str, set[str]]:
m = re.match(r"^(.+)(\[[^\]]+\])$", path)
extras = None
if m:
path_no_extras = m.group(1).rstrip()
extras = m.group(2)
else:
path_no_extras = path
return path_no_extras, _convert_extras(extras)
@@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import os
import re
import site
import sys
logger = logging.getLogger(__name__)
@@ -12,7 +11,7 @@ _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile(
)
def _running_under_venv() -> bool:
def running_under_virtualenv() -> bool:
"""Checks if sys.base_prefix and sys.prefix match.
This handles PEP 405 compliant virtual environments.
@@ -20,20 +19,6 @@ def _running_under_venv() -> bool:
return sys.prefix != getattr(sys, "base_prefix", sys.prefix)
def _running_under_legacy_virtualenv() -> bool:
"""Checks if sys.real_prefix is set.
This handles virtual environments created with pypa's virtualenv.
"""
# pypa/virtualenv case
return hasattr(sys, "real_prefix")
def running_under_virtualenv() -> bool:
"""True if we're running inside a virtual environment, False otherwise."""
return _running_under_venv() or _running_under_legacy_virtualenv()
def _get_pyvenv_cfg_lines() -> list[str] | None:
"""Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines
@@ -78,28 +63,9 @@ def _no_global_under_venv() -> bool:
return False
def _no_global_under_legacy_virtualenv() -> bool:
"""Check if "no-global-site-packages.txt" exists beside site.py
This mirrors logic in pypa/virtualenv for determining whether system
site-packages are visible in the virtual environment.
"""
site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
no_global_site_packages_file = os.path.join(
site_mod_dir,
"no-global-site-packages.txt",
)
return os.path.exists(no_global_site_packages_file)
def virtualenv_no_global() -> bool:
"""Returns a boolean, whether running in venv with no system site-packages."""
# PEP 405 compliance needs to be checked first since virtualenv >=20 would
# return True for both checks, but is only able to use the PEP 405 config.
if _running_under_venv():
if running_under_virtualenv():
return _no_global_under_venv()
if _running_under_legacy_virtualenv():
return _no_global_under_legacy_virtualenv()
return False
@@ -5,12 +5,11 @@ import os.path
import pathlib
import re
import urllib.parse
import urllib.request
from dataclasses import replace
from typing import Any
from pip._internal.exceptions import BadCommand, InstallationError
from pip._internal.utils.misc import HiddenText, display_path, hide_url
from pip._internal.utils.misc import HiddenText, display_path, hide_url, strtobool
from pip._internal.utils.subprocess import make_command
from pip._internal.vcs.versioncontrol import (
AuthInfo,
@@ -281,7 +280,9 @@ class Git(VersionControl):
flags = ()
else:
flags = ("--verbose", "--progress")
if self.get_git_version() >= (2, 17):
if self.get_git_version() >= (2, 17) and not strtobool(
os.environ.get("PIP_NO_PARTIAL_CLONE_FOR_BROKEN_GIT_SERVER", "no")
):
# Git added support for partial clone in 2.17
# https://git-scm.com/docs/partial-clone
# Speeds up cloning by functioning without a complete copy of repository
@@ -438,7 +439,7 @@ class Git(VersionControl):
if os.path.exists(url):
# A local bare remote (git clone --mirror).
# Needs a file:// prefix.
return pathlib.PurePath(url).as_uri()
return pathlib.Path(url).as_uri()
scp_match = SCP_REGEX.match(url)
if scp_match:
# Add an ssh:// prefix and replace the ':' with a '/'.
@@ -453,7 +454,7 @@ class Git(VersionControl):
"""
try:
cls.run_command(
["rev-parse", "-q", "--verify", "sha^" + rev],
["rev-parse", "-q", "--verify", rev + "^{commit}"],
cwd=location,
log_failed_cmd=False,
)
@@ -500,6 +501,8 @@ class Git(VersionControl):
work with a ssh:// scheme (e.g. GitHub). But we need a scheme for
parsing. Hence we remove it again afterwards and return it as a stub.
"""
import urllib.request
# Works around an apparent Git bug
# (see https://article.gmane.org/gmane.comp.version-control.git/146500)
scheme, netloc, path, query, fragment = urlsplit(url)
@@ -12,7 +12,6 @@ from dataclasses import dataclass, field
from typing import (
Any,
Literal,
Optional,
)
from pip._internal.cli.spinners import SpinnerInterface
@@ -39,7 +38,7 @@ __all__ = ["vcs"]
logger = logging.getLogger(__name__)
AuthInfo = tuple[Optional[str], Optional[str]]
AuthInfo = tuple[str | None, str | None]
def is_url(name: str) -> bool:
@@ -49,6 +49,9 @@ def _should_cache(
if req.editable or not req.source_dir:
# never cache editable requirements
return False
if req.link and req.link.is_existing_dir():
# never cache local directory requirements
return False
if req.link and req.link.is_vcs:
# VCS checkout. Do not cache
@@ -205,6 +208,7 @@ def build(
requirements: Iterable[InstallRequirement],
wheel_cache: WheelCache,
verify: bool,
allow_editables: bool,
) -> BuildResult:
"""Build wheels.
@@ -229,7 +233,7 @@ def build(
req,
cache_dir,
verify,
req.editable and req.permit_editable_wheels,
req.editable and allow_editables,
)
if wheel_file:
# Record the download origin in the cache