bro
This commit is contained in:
1 parent
de41a02749
commit
9af09cebd3
764 files changed
+54218
-78859
No files matched your search
@@ -3,12 +3,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Generator, Iterable
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from contextlib import suppress
|
||||
from email.parser import Parser
|
||||
from functools import reduce
|
||||
from typing import (
|
||||
Callable,
|
||||
NamedTuple,
|
||||
)
|
||||
|
||||
@@ -168,7 +167,7 @@ def _create_whitelist(
|
||||
continue
|
||||
|
||||
for req in package_set[package_name].dependencies:
|
||||
if canonicalize_name(req.name) in packages_affected:
|
||||
if canonicalize_name(req.name) in would_be_installed:
|
||||
packages_affected.add(package_name)
|
||||
break
|
||||
|
||||
|
||||
@@ -15,26 +15,25 @@ import sys
|
||||
import textwrap
|
||||
import warnings
|
||||
from base64 import urlsafe_b64encode
|
||||
from collections.abc import Generator, Iterable, Iterator, Sequence
|
||||
from collections.abc import Callable, Generator, Iterable, Iterator, Sequence
|
||||
from email.message import Message
|
||||
from itertools import chain, filterfalse, starmap
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
IO,
|
||||
Any,
|
||||
BinaryIO,
|
||||
Callable,
|
||||
NewType,
|
||||
Protocol,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
from zipfile import ZipFile, ZipInfo
|
||||
|
||||
from pip._vendor.distlib.scripts import ScriptMaker
|
||||
from pip._vendor.distlib.util import get_export_entry
|
||||
from pip._vendor.distlib.util import FileOperator, get_export_entry
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal.exceptions import InstallationError
|
||||
from pip._internal.exceptions import BlockedImportError, InstallationError
|
||||
from pip._internal.locations import get_major_minor_version
|
||||
from pip._internal.metadata import (
|
||||
BaseDistribution,
|
||||
@@ -59,6 +58,10 @@ class File(Protocol):
|
||||
dest_path: str
|
||||
changed: bool
|
||||
|
||||
@property
|
||||
def written(self) -> bool:
|
||||
pass
|
||||
|
||||
def save(self) -> None:
|
||||
pass
|
||||
|
||||
@@ -66,7 +69,7 @@ class File(Protocol):
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RecordPath = NewType("RecordPath", str)
|
||||
InstalledCSVRow = tuple[RecordPath, str, Union[int, str]]
|
||||
InstalledCSVRow = tuple[RecordPath, str, int | str]
|
||||
|
||||
|
||||
def rehash(path: str, blocksize: int = 1 << 20) -> tuple[str, str]:
|
||||
@@ -127,27 +130,33 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> str | None:
|
||||
return None
|
||||
|
||||
# Group scripts by the path they were installed in
|
||||
grouped_by_dir: dict[str, set[str]] = collections.defaultdict(set)
|
||||
grouped_by_dir: dict[Path, set[str]] = collections.defaultdict(set)
|
||||
for destfile in scripts:
|
||||
parent_dir = os.path.dirname(destfile)
|
||||
script_name = os.path.basename(destfile)
|
||||
dest_path = Path(destfile)
|
||||
parent_dir = dest_path.parent.resolve()
|
||||
script_name = dest_path.name
|
||||
grouped_by_dir[parent_dir].add(script_name)
|
||||
|
||||
# We don't want to warn for directories that are on PATH.
|
||||
not_warn_dirs = [
|
||||
os.path.normcase(os.path.normpath(i)).rstrip(os.sep)
|
||||
for i in os.environ.get("PATH", "").split(os.pathsep)
|
||||
]
|
||||
# If an executable sits with sys.executable, we don't warn for it.
|
||||
# This covers the case of venv invocations without activating the venv.
|
||||
not_warn_dirs.append(
|
||||
os.path.normcase(os.path.normpath(os.path.dirname(sys.executable)))
|
||||
)
|
||||
warn_for: dict[str, set[str]] = {
|
||||
executable_dir = Path(sys.executable).parent.resolve()
|
||||
path_entries = os.environ.get("PATH", "").split(os.pathsep)
|
||||
|
||||
# We don't want to warn for directories that are on PATH.
|
||||
warn_for: dict[Path, set[str]] = {
|
||||
parent_dir: scripts
|
||||
for parent_dir, scripts in grouped_by_dir.items()
|
||||
if os.path.normcase(os.path.normpath(parent_dir)) not in not_warn_dirs
|
||||
if parent_dir != executable_dir and str(parent_dir) not in path_entries
|
||||
}
|
||||
|
||||
# Resolving PATH entries can be slow, so only do it for what's left.
|
||||
if warn_for:
|
||||
not_warn_dirs = {Path(i).resolve() for i in path_entries}
|
||||
warn_for = {
|
||||
parent_dir: scripts
|
||||
for parent_dir, scripts in warn_for.items()
|
||||
if parent_dir not in not_warn_dirs
|
||||
}
|
||||
if not warn_for:
|
||||
return None
|
||||
|
||||
@@ -176,9 +185,7 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> str | None:
|
||||
msg_lines.append(last_line_fmt.format("these directories"))
|
||||
|
||||
# Add a note if any directory starts with ~
|
||||
warn_for_tilde = any(
|
||||
i[0] == "~" for i in os.environ.get("PATH", "").split(os.pathsep) if i
|
||||
)
|
||||
warn_for_tilde = any(i[0] == "~" for i in path_entries if i)
|
||||
if warn_for_tilde:
|
||||
tilde_warning_msg = (
|
||||
"NOTE: The current PATH contains path(s) starting with `~`, "
|
||||
@@ -345,6 +352,7 @@ class ZipBackedFile:
|
||||
self.dest_path = dest_path
|
||||
self._zip_file = zip_file
|
||||
self.changed = False
|
||||
self.written = False
|
||||
|
||||
def _getinfo(self) -> ZipInfo:
|
||||
return self._zip_file.getinfo(self.src_record_path)
|
||||
@@ -366,6 +374,7 @@ class ZipBackedFile:
|
||||
# optimization: the file is created by open(),
|
||||
# skip the decompression when there is 0 bytes to decompress.
|
||||
with open(self.dest_path, "wb") as dest:
|
||||
self.written = True
|
||||
if zipinfo.file_size > 0:
|
||||
with self._zip_file.open(zipinfo) as f:
|
||||
blocksize = min(zipinfo.file_size, 1024 * 1024)
|
||||
@@ -382,6 +391,10 @@ class ScriptFile:
|
||||
self.dest_path = self._file.dest_path
|
||||
self.changed = False
|
||||
|
||||
@property
|
||||
def written(self) -> bool:
|
||||
return self._file.written
|
||||
|
||||
def save(self) -> None:
|
||||
self._file.save()
|
||||
self.changed = fix_script(self.dest_path)
|
||||
@@ -397,32 +410,171 @@ class MissingCallableSuffix(InstallationError):
|
||||
)
|
||||
|
||||
|
||||
def _raise_for_invalid_entrypoint(specification: str) -> None:
|
||||
def _raise_for_invalid_entrypoint(specification: str, scripts_dir: str) -> None:
|
||||
entry = get_export_entry(specification)
|
||||
if entry is not None and entry.suffix is None:
|
||||
if entry is None:
|
||||
return
|
||||
|
||||
if entry.suffix is None:
|
||||
raise MissingCallableSuffix(str(entry))
|
||||
|
||||
# distlib joins the entry point name onto the scripts directory, so a name
|
||||
# with path separators or ``..`` components can resolve elsewhere. The script
|
||||
# must resolve to a path strictly inside the scripts directory.
|
||||
dest = os.path.join(scripts_dir, entry.name)
|
||||
resolves_to_scripts_dir = os.path.abspath(dest) == os.path.abspath(scripts_dir)
|
||||
if resolves_to_scripts_dir or not is_within_directory(scripts_dir, dest):
|
||||
raise InstallationError(
|
||||
f"Invalid script entry point name {entry.name!r}: the script "
|
||||
f"would be installed outside the scripts directory ({scripts_dir})."
|
||||
)
|
||||
|
||||
|
||||
class PipScriptMaker(ScriptMaker):
|
||||
# Override distlib's default script template with one that
|
||||
# doesn't import `re` module, allowing scripts to load faster.
|
||||
script_template = textwrap.dedent(
|
||||
"""\
|
||||
script_template = textwrap.dedent("""\
|
||||
import sys
|
||||
from %(module)s import %(import_name)s
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = sys.argv[0].removesuffix('.exe')
|
||||
sys.exit(%(func)s())
|
||||
"""
|
||||
)
|
||||
""")
|
||||
|
||||
def make(
|
||||
self, specification: str, options: dict[str, Any] | None = None
|
||||
) -> list[str]:
|
||||
_raise_for_invalid_entrypoint(specification)
|
||||
_raise_for_invalid_entrypoint(specification, self.target_dir)
|
||||
return super().make(specification, options)
|
||||
|
||||
|
||||
class _WheelInstallRecord:
|
||||
"""Track files and metadata needed to recover an interrupted installation."""
|
||||
|
||||
def __init__(self, info_dir: str, lib_dir: str) -> None:
|
||||
self.info_dir = info_dir
|
||||
self.lib_dir = lib_dir
|
||||
self.installed: dict[RecordPath, RecordPath] = {}
|
||||
self.generated: list[str] = []
|
||||
self.script_fileop = FileOperator()
|
||||
self.script_fileop.record = True
|
||||
self.started = False
|
||||
|
||||
def written_paths(self) -> set[str]:
|
||||
"""Return the files written so far, including partial copies and scripts."""
|
||||
paths = {os.path.join(self.lib_dir, path) for path in self.installed.values()}
|
||||
paths.update(self.generated)
|
||||
paths.update(self.script_fileop.files_written)
|
||||
return paths
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _generate_file(
|
||||
path: str, file_mode: int, **kwargs: Any
|
||||
) -> Generator[BinaryIO, None, None]:
|
||||
"""Write metadata atomically with the requested permissions."""
|
||||
temporary_path = None
|
||||
try:
|
||||
with adjacent_tmp_file(path, **kwargs) as f:
|
||||
temporary_path = f.name
|
||||
yield f
|
||||
|
||||
os.chmod(f.name, file_mode)
|
||||
replace(f.name, path)
|
||||
except BaseException:
|
||||
if temporary_path is not None:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(temporary_path)
|
||||
raise
|
||||
|
||||
|
||||
def _write_recovery_record(wheel_zip: ZipFile, record: _WheelInstallRecord) -> None:
|
||||
"""Write readable metadata and an uninstall record for files already written."""
|
||||
info_dir, lib_dir = record.info_dir, record.lib_dir
|
||||
file_mode = 0o666 & ~current_umask()
|
||||
|
||||
recovery_paths = {
|
||||
_fs_to_record_path(path, lib_dir) for path in record.written_paths()
|
||||
}
|
||||
|
||||
dest_info_dir = os.path.join(lib_dir, info_dir)
|
||||
ensure_dir(dest_info_dir)
|
||||
metadata_path = os.path.join(dest_info_dir, "METADATA")
|
||||
with _generate_file(metadata_path, file_mode) as metadata_file:
|
||||
metadata_file.write(wheel_zip.read(f"{info_dir}/METADATA"))
|
||||
recovery_paths.add(_fs_to_record_path(metadata_path, lib_dir))
|
||||
|
||||
# Uninstall must not infer wrappers that were never generated.
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(os.path.join(dest_info_dir, "entry_points.txt"))
|
||||
|
||||
record_path = os.path.join(dest_info_dir, "RECORD")
|
||||
recovery_paths.add(_fs_to_record_path(record_path, lib_dir))
|
||||
with _generate_file(record_path, file_mode, **csv_io_kwargs("w")) as record_file:
|
||||
writer = csv.writer(cast("IO[str]", record_file))
|
||||
writer.writerows((path, "", "") for path in sorted(recovery_paths))
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _record_wheel_installation(
|
||||
wheel_zip: ZipFile, name: str, scheme: Scheme, *, rollback_on_error: bool = False
|
||||
) -> Generator[_WheelInstallRecord, None, None]:
|
||||
"""Provide the file journal and recover writes interrupted by blocked imports."""
|
||||
info_dir, metadata = parse_wheel(wheel_zip, name)
|
||||
lib_dir = scheme.purelib if wheel_root_is_purelib(metadata) else scheme.platlib
|
||||
record = _WheelInstallRecord(info_dir, lib_dir)
|
||||
|
||||
try:
|
||||
yield record
|
||||
except BlockedImportError as error:
|
||||
# A blocked preparation step must leave an existing install alone.
|
||||
if not record.started:
|
||||
raise
|
||||
|
||||
if rollback_on_error:
|
||||
# Remove replacement files before the previous installation is restored.
|
||||
try:
|
||||
for path in record.written_paths():
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(path)
|
||||
|
||||
# Empty dist-info directories still look like installed distributions.
|
||||
for directory, _, _ in os.walk(
|
||||
os.path.join(lib_dir, info_dir), topdown=False
|
||||
):
|
||||
with contextlib.suppress(OSError):
|
||||
os.rmdir(directory)
|
||||
except (OSError, BlockedImportError) as cleanup_error:
|
||||
raise BlockedImportError(
|
||||
f"Could not remove incomplete installation of {name!r}: "
|
||||
f"{cleanup_error}\n{error}"
|
||||
) from cleanup_error
|
||||
|
||||
raise
|
||||
|
||||
try:
|
||||
_write_recovery_record(wheel_zip, record)
|
||||
except (OSError, BlockedImportError) as recovery_error:
|
||||
raise BlockedImportError(
|
||||
f"Could not write recovery metadata for {name!r} at "
|
||||
f"{os.path.join(lib_dir, info_dir)!r}: {recovery_error}\n{error}"
|
||||
) from recovery_error
|
||||
|
||||
raise BlockedImportError(
|
||||
f"Installation of {name!r} was interrupted and the package may be "
|
||||
f"corrupt. Run 'pip uninstall {name}' to remove it. {error}"
|
||||
) from error
|
||||
|
||||
|
||||
def _compile_wheel_file(path: str) -> bool:
|
||||
"""Skip optional compilation when it requires a blocked or unavailable import."""
|
||||
try:
|
||||
return compileall.compile_file(path, force=True, quiet=True)
|
||||
except (BlockedImportError, ImportError) as error:
|
||||
logger.debug("Skipping bytecode compilation for %s: %s", path, error)
|
||||
return False
|
||||
|
||||
|
||||
def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
name: str,
|
||||
wheel_zip: ZipFile,
|
||||
@@ -432,36 +584,33 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
warn_script_location: bool = True,
|
||||
direct_url: DirectUrl | None = None,
|
||||
requested: bool = False,
|
||||
script_executable: str | None = None,
|
||||
*,
|
||||
record: _WheelInstallRecord,
|
||||
) -> None:
|
||||
"""Install a wheel.
|
||||
|
||||
:param name: Name of the project to install
|
||||
:param wheel_zip: open ZipFile for wheel being installed
|
||||
:param scheme: Distutils scheme dictating the install directories
|
||||
:param req_description: String used in place of the requirement, for
|
||||
logging
|
||||
:param pycompile: Whether to byte-compile installed Python files
|
||||
:param warn_script_location: Whether to check that scripts are installed
|
||||
into a directory on PATH
|
||||
:param script_executable: Python executable to use for console scripts
|
||||
:raises UnsupportedWheel:
|
||||
* when the directory holds an unpacked wheel with incompatible
|
||||
Wheel-Version
|
||||
* when the .dist-info dir does not match the wheel
|
||||
"""
|
||||
info_dir, metadata = parse_wheel(wheel_zip, name)
|
||||
|
||||
if wheel_root_is_purelib(metadata):
|
||||
lib_dir = scheme.purelib
|
||||
else:
|
||||
lib_dir = scheme.platlib
|
||||
info_dir, lib_dir = record.info_dir, record.lib_dir
|
||||
|
||||
# Record details of the files moved
|
||||
# installed = files copied from the wheel to the destination
|
||||
# changed = files changed while installing (scripts #! line typically)
|
||||
# generated = files newly generated during the install (script wrappers)
|
||||
installed: dict[RecordPath, RecordPath] = {}
|
||||
# generated = scripts and installation metadata generated during the install
|
||||
installed = record.installed
|
||||
changed: set[RecordPath] = set()
|
||||
generated: list[str] = []
|
||||
generated = record.generated
|
||||
|
||||
def record_installed(
|
||||
srcfile: RecordPath, destfile: str, modified: bool = False
|
||||
@@ -582,6 +731,8 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
script_scheme_files = map(ScriptFile, script_scheme_files)
|
||||
files = chain(files, script_scheme_files)
|
||||
|
||||
# File saves may unlink old files before creating replacements.
|
||||
record.started = True
|
||||
existing_parents = set()
|
||||
for file in files:
|
||||
# directory creation is lazy and after file filtering
|
||||
@@ -591,7 +742,14 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
if parent_dir not in existing_parents:
|
||||
ensure_dir(parent_dir)
|
||||
existing_parents.add(parent_dir)
|
||||
file.save()
|
||||
|
||||
try:
|
||||
file.save()
|
||||
except BlockedImportError:
|
||||
if file.written:
|
||||
record_installed(file.src_record_path, file.dest_path, True)
|
||||
raise
|
||||
|
||||
record_installed(file.src_record_path, file.dest_path, file.changed)
|
||||
|
||||
def pyc_source_file_paths() -> Generator[str, None, None]:
|
||||
@@ -619,7 +777,7 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore")
|
||||
for path in pyc_source_file_paths():
|
||||
success = compileall.compile_file(path, force=True, quiet=True)
|
||||
success = _compile_wheel_file(path)
|
||||
if success:
|
||||
pyc_path = pyc_output_path(path)
|
||||
assert os.path.exists(pyc_path)
|
||||
@@ -629,7 +787,14 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
record_installed(pyc_record_path, pyc_path)
|
||||
logger.debug(stdout.getvalue())
|
||||
|
||||
maker = PipScriptMaker(None, scheme.scripts)
|
||||
# Track scripts even if generation fails before returning their paths.
|
||||
maker = PipScriptMaker(None, scheme.scripts, fileop=record.script_fileop)
|
||||
|
||||
# Embed the target environment's interpreter in console-script launchers
|
||||
# rather than the one running pip, so an in-process install into another
|
||||
# environment (e.g. a venv build environment) produces working launchers.
|
||||
if script_executable is not None:
|
||||
maker.executable = script_executable # type: ignore # it's untyped in distlib
|
||||
|
||||
# Ensure old scripts are overwritten.
|
||||
# See https://github.com/pypa/pip/issues/1800
|
||||
@@ -661,26 +826,18 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
logger.warning(msg)
|
||||
|
||||
generated_file_mode = 0o666 & ~current_umask()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _generate_file(path: str, **kwargs: Any) -> Generator[BinaryIO, None, None]:
|
||||
with adjacent_tmp_file(path, **kwargs) as f:
|
||||
yield f
|
||||
os.chmod(f.name, generated_file_mode)
|
||||
replace(f.name, path)
|
||||
|
||||
dest_info_dir = os.path.join(lib_dir, info_dir)
|
||||
|
||||
# Record pip as the installer
|
||||
installer_path = os.path.join(dest_info_dir, "INSTALLER")
|
||||
with _generate_file(installer_path) as installer_file:
|
||||
with _generate_file(installer_path, generated_file_mode) as installer_file:
|
||||
installer_file.write(b"pip\n")
|
||||
generated.append(installer_path)
|
||||
|
||||
# Record the PEP 610 direct URL reference
|
||||
if direct_url is not None:
|
||||
direct_url_path = os.path.join(dest_info_dir, DIRECT_URL_METADATA_NAME)
|
||||
with _generate_file(direct_url_path) as direct_url_file:
|
||||
with _generate_file(direct_url_path, generated_file_mode) as direct_url_file:
|
||||
direct_url_file.write(direct_url.to_json().encode("utf-8"))
|
||||
generated.append(direct_url_path)
|
||||
|
||||
@@ -694,9 +851,10 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
record_text = distribution.read_text("RECORD")
|
||||
record_rows = list(csv.reader(record_text.splitlines()))
|
||||
|
||||
# RECORD generation consumes its input; recovery still needs the journal.
|
||||
rows = get_csv_rows_for_installed(
|
||||
record_rows,
|
||||
installed=installed,
|
||||
installed=installed.copy(),
|
||||
changed=changed,
|
||||
generated=generated,
|
||||
lib_dir=lib_dir,
|
||||
@@ -705,7 +863,9 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long
|
||||
# Record details of all files installed
|
||||
record_path = os.path.join(dest_info_dir, "RECORD")
|
||||
|
||||
with _generate_file(record_path, **csv_io_kwargs("w")) as record_file:
|
||||
with _generate_file(
|
||||
record_path, generated_file_mode, **csv_io_kwargs("w")
|
||||
) as record_file:
|
||||
# Explicitly cast to typing.IO[str] as a workaround for the mypy error:
|
||||
# "writer" has incompatible type "BinaryIO"; expected "_Writer"
|
||||
writer = csv.writer(cast("IO[str]", record_file))
|
||||
@@ -730,9 +890,17 @@ def install_wheel(
|
||||
warn_script_location: bool = True,
|
||||
direct_url: DirectUrl | None = None,
|
||||
requested: bool = False,
|
||||
script_executable: str | None = None,
|
||||
*,
|
||||
rollback_on_error: bool = False,
|
||||
) -> None:
|
||||
with ZipFile(wheel_path, allowZip64=True) as z:
|
||||
with req_error_context(req_description):
|
||||
with (
|
||||
req_error_context(req_description),
|
||||
_record_wheel_installation(
|
||||
z, name, scheme, rollback_on_error=rollback_on_error
|
||||
) as record,
|
||||
):
|
||||
_install_wheel(
|
||||
name=name,
|
||||
wheel_zip=z,
|
||||
@@ -742,4 +910,6 @@ def install_wheel(
|
||||
warn_script_location=warn_script_location,
|
||||
direct_url=direct_url,
|
||||
requested=requested,
|
||||
script_executable=script_executable,
|
||||
record=record,
|
||||
)
|
||||
@@ -12,9 +12,10 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
|
||||
from pip._internal.build_env import BuildEnvironmentInstaller
|
||||
from pip._internal.build_env import BuildEnvironmentInstaller, BuildIsolationMode
|
||||
from pip._internal.distributions import make_distribution_for_install_requirement
|
||||
from pip._internal.distributions.installed import InstalledDistribution
|
||||
from pip._internal.exceptions import (
|
||||
@@ -23,13 +24,16 @@ from pip._internal.exceptions import (
|
||||
HashUnpinned,
|
||||
InstallationError,
|
||||
MetadataInconsistent,
|
||||
MetadataInvalid,
|
||||
NetworkConnectionError,
|
||||
SidecarMetadataInconsistent,
|
||||
VcsHashUnsupported,
|
||||
)
|
||||
from pip._internal.index.package_finder import PackageFinder
|
||||
from pip._internal.metadata import BaseDistribution, get_metadata_distribution
|
||||
from pip._internal.models.direct_url import ArchiveInfo
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.metadata.direct_references import check_pypi_direct_references
|
||||
from pip._internal.models.direct_url import ArchiveInfo, DirectUrl
|
||||
from pip._internal.models.link import Link, join_within_directory
|
||||
from pip._internal.models.wheel import Wheel
|
||||
from pip._internal.network.download import Downloader
|
||||
from pip._internal.network.lazy_wheel import (
|
||||
@@ -52,6 +56,7 @@ from pip._internal.utils.misc import (
|
||||
hide_url,
|
||||
redact_auth_from_requirement,
|
||||
)
|
||||
from pip._internal.utils.packaging import get_requirement
|
||||
from pip._internal.utils.temp_dir import TempDirectory
|
||||
from pip._internal.utils.unpacking import unpack_file
|
||||
from pip._internal.vcs import vcs
|
||||
@@ -66,8 +71,9 @@ def _get_prepared_distribution(
|
||||
req: InstallRequirement,
|
||||
build_tracker: BuildTracker,
|
||||
build_env_installer: BuildEnvironmentInstaller,
|
||||
build_isolation: bool,
|
||||
build_isolation: BuildIsolationMode,
|
||||
check_build_deps: bool,
|
||||
allow_editables: bool,
|
||||
) -> BaseDistribution:
|
||||
"""Prepare a distribution for installation."""
|
||||
abstract_dist = make_distribution_for_install_requirement(req)
|
||||
@@ -75,11 +81,22 @@ def _get_prepared_distribution(
|
||||
if tracker_id is not None:
|
||||
with build_tracker.track(req, tracker_id):
|
||||
abstract_dist.prepare_distribution_metadata(
|
||||
build_env_installer, build_isolation, check_build_deps
|
||||
build_env_installer, build_isolation, check_build_deps, allow_editables
|
||||
)
|
||||
return abstract_dist.get_metadata_distribution()
|
||||
|
||||
|
||||
def _check_linked_requirement_metadata(
|
||||
req: InstallRequirement,
|
||||
distribution: BaseDistribution,
|
||||
) -> None:
|
||||
"""Apply source-dependent checks to prepared metadata."""
|
||||
source_link = req.source_link
|
||||
if source_link is None:
|
||||
raise InstallationError(f"Cannot prepare {req}: the source link is missing.")
|
||||
check_pypi_direct_references(distribution, source_link)
|
||||
|
||||
|
||||
def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
|
||||
vcs_backend = vcs.get_backend_for_scheme(link.scheme)
|
||||
assert vcs_backend is not None
|
||||
@@ -201,7 +218,7 @@ def _check_download_dir(
|
||||
"""Check download_dir for previously downloaded file with correct hash
|
||||
If a correct file is found return its path else None
|
||||
"""
|
||||
download_path = os.path.join(download_dir, link.filename)
|
||||
download_path = join_within_directory(download_dir, link.filename)
|
||||
|
||||
if not os.path.exists(download_path):
|
||||
return None
|
||||
@@ -222,6 +239,91 @@ def _check_download_dir(
|
||||
return download_path
|
||||
|
||||
|
||||
def _canonical_requires(
|
||||
req: InstallRequirement, dist: BaseDistribution, source: str
|
||||
) -> frozenset[Requirement]:
|
||||
"""Return the canonicalized ``Requires-Dist`` entries of ``dist``.
|
||||
|
||||
``source`` describes which metadata file ``dist`` was parsed from, for
|
||||
use in error messages.
|
||||
"""
|
||||
canonical = set()
|
||||
for raw in dist.iter_raw_dependencies():
|
||||
try:
|
||||
# strip() because a folded metadata header may be returned
|
||||
# with a leading newline; iter_dependencies() strips for the
|
||||
# same reason.
|
||||
canonical.add(get_requirement(raw.strip()))
|
||||
except InvalidRequirement as e:
|
||||
raise MetadataInvalid(req, f"Requires-Dist in {source}: {e}")
|
||||
return frozenset(canonical)
|
||||
|
||||
|
||||
def _check_sidecar_matches_wheel(
|
||||
req: InstallRequirement,
|
||||
sidecar_dist: BaseDistribution,
|
||||
wheel_dist: BaseDistribution,
|
||||
) -> None:
|
||||
"""Check that a .metadata-based distribution matches the wheel's METADATA.
|
||||
|
||||
Compare ``Name``, ``Version``, ``Requires-Dist``, ``Requires-Python``
|
||||
and ``Provides-Extra`` between the two and abort the install on any
|
||||
mismatch as PEP 658 requires the metadata files "MUST be identical".
|
||||
|
||||
While the PEP doesn't mandate that consumers enforce the identical
|
||||
requirement, it's good nonetheless to check to prevent confusing
|
||||
behaviour when an index misbehaves.
|
||||
|
||||
Also note for name and version, pip usually rejects wheels if they're
|
||||
inconsistent already. Checking them again here is purely defensive.
|
||||
"""
|
||||
|
||||
sidecar_name = canonicalize_name(sidecar_dist.raw_name)
|
||||
wheel_name = canonicalize_name(wheel_dist.raw_name)
|
||||
if sidecar_name != wheel_name:
|
||||
raise SidecarMetadataInconsistent(req, "Name", sidecar_name, wheel_name)
|
||||
|
||||
if sidecar_dist.version != wheel_dist.version:
|
||||
raise SidecarMetadataInconsistent(
|
||||
req,
|
||||
"Version",
|
||||
str(sidecar_dist.version),
|
||||
str(wheel_dist.version),
|
||||
)
|
||||
|
||||
# For multi-use fields, only report the symmetric difference to avoid
|
||||
# unnecessarily flagging matching values.
|
||||
sidecar_requires = _canonical_requires(
|
||||
req, sidecar_dist, "the PEP 658 .metadata file"
|
||||
)
|
||||
wheel_requires = _canonical_requires(req, wheel_dist, "the wheel's METADATA")
|
||||
if sidecar_requires != wheel_requires:
|
||||
raise SidecarMetadataInconsistent(
|
||||
req,
|
||||
"Requires-Dist",
|
||||
", ".join(sorted(map(str, sidecar_requires - wheel_requires))),
|
||||
", ".join(sorted(map(str, wheel_requires - sidecar_requires))),
|
||||
)
|
||||
|
||||
if sidecar_dist.requires_python != wheel_dist.requires_python:
|
||||
raise SidecarMetadataInconsistent(
|
||||
req,
|
||||
"Requires-Python",
|
||||
str(sidecar_dist.requires_python),
|
||||
str(wheel_dist.requires_python),
|
||||
)
|
||||
|
||||
sidecar_extras = frozenset(sidecar_dist.iter_provided_extras())
|
||||
wheel_extras = frozenset(wheel_dist.iter_provided_extras())
|
||||
if sidecar_extras != wheel_extras:
|
||||
raise SidecarMetadataInconsistent(
|
||||
req,
|
||||
"Provides-Extra",
|
||||
", ".join(sorted(sidecar_extras - wheel_extras)),
|
||||
", ".join(sorted(wheel_extras - sidecar_extras)),
|
||||
)
|
||||
|
||||
|
||||
class RequirementPreparer:
|
||||
"""Prepares a Requirement"""
|
||||
|
||||
@@ -231,7 +333,7 @@ class RequirementPreparer:
|
||||
build_dir: str,
|
||||
download_dir: str | None,
|
||||
src_dir: str,
|
||||
build_isolation: bool,
|
||||
build_isolation: BuildIsolationMode,
|
||||
build_isolation_installer: BuildEnvironmentInstaller,
|
||||
check_build_deps: bool,
|
||||
build_tracker: BuildTracker,
|
||||
@@ -243,6 +345,7 @@ class RequirementPreparer:
|
||||
lazy_wheel: bool,
|
||||
verbosity: int,
|
||||
legacy_resolver: bool,
|
||||
allow_editables: bool,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -285,6 +388,10 @@ class RequirementPreparer:
|
||||
# Previous "header" printed for a link-based InstallRequirement
|
||||
self._previous_requirement_header = ("", "")
|
||||
|
||||
# Do we allow preparing editable wheels?
|
||||
# When using pip wheel, editables must still produce regular wheels
|
||||
self.allow_editables = allow_editables
|
||||
|
||||
def _log_preparing_link(self, req: InstallRequirement) -> None:
|
||||
"""Provide context for the requirement being prepared."""
|
||||
if req.link.is_file and not req.is_wheel_from_cache:
|
||||
@@ -342,7 +449,8 @@ class RequirementPreparer:
|
||||
# (For example, we can raise VcsHashUnsupported for a VCS URL
|
||||
# rather than HashMissing.)
|
||||
if not self.require_hashes:
|
||||
return req.hashes(trust_internet=True)
|
||||
# An index-provided hash must not override one supplied by the user.
|
||||
return req.hashes(trust_internet=not req.has_hash_options)
|
||||
|
||||
# We could check these first 2 conditions inside unpack_url
|
||||
# and save repetition of conditions, but then we would
|
||||
@@ -529,6 +637,7 @@ class RequirementPreparer:
|
||||
# The file is not available, attempt to fetch only metadata
|
||||
metadata_dist = self._fetch_metadata_only(req)
|
||||
if metadata_dist is not None:
|
||||
_check_linked_requirement_metadata(req, metadata_dist)
|
||||
req.needs_more_preparation = True
|
||||
req.set_dist(metadata_dist)
|
||||
# Ensure download_info is available even in dry-run mode
|
||||
@@ -586,9 +695,9 @@ class RequirementPreparer:
|
||||
# We need to verify hashes, and we have found the requirement in the cache
|
||||
# of locally built wheels.
|
||||
if (
|
||||
isinstance(req.download_info.info, ArchiveInfo)
|
||||
and req.download_info.info.hashes
|
||||
and hashes.has_one_of(req.download_info.info.hashes)
|
||||
req.download_info.archive_info
|
||||
and req.download_info.archive_info.hashes
|
||||
and hashes.has_one_of(req.download_info.archive_info.hashes)
|
||||
):
|
||||
# At this point we know the requirement was built from a hashable source
|
||||
# artifact, and we verified that the cache entry's hash of the original
|
||||
@@ -640,14 +749,18 @@ class RequirementPreparer:
|
||||
# compute it from the downloaded file.
|
||||
# FIXME: https://github.com/pypa/pip/issues/11943
|
||||
if (
|
||||
isinstance(req.download_info.info, ArchiveInfo)
|
||||
and not req.download_info.info.hashes
|
||||
req.download_info.archive_info
|
||||
and not req.download_info.archive_info.hashes
|
||||
and local_file
|
||||
):
|
||||
hash = hash_file(local_file.path)[0].hexdigest()
|
||||
# We populate info.hash for backward compatibility.
|
||||
# This will automatically populate info.hashes.
|
||||
req.download_info.info.hash = f"sha256={hash}"
|
||||
# We populate archive_info.hashes. For backward compatibility,
|
||||
# the legacy hash field will be generated when converting to JSON.
|
||||
req.download_info = DirectUrl(
|
||||
url=req.download_info.url,
|
||||
archive_info=ArchiveInfo(hashes={"sha256": hash}),
|
||||
subdirectory=req.download_info.subdirectory,
|
||||
)
|
||||
|
||||
# For use in later processing,
|
||||
# preserve the file path on the requirement.
|
||||
@@ -660,7 +773,30 @@ class RequirementPreparer:
|
||||
self.build_env_installer,
|
||||
self.build_isolation,
|
||||
self.check_build_deps,
|
||||
self.allow_editables,
|
||||
)
|
||||
_check_linked_requirement_metadata(req, dist)
|
||||
|
||||
# If a PEP 658 .metadata file was used, check that fields relevant for
|
||||
# dependency resolution match with the wheel's METADATA file.
|
||||
#
|
||||
# NOTE: PEP 658 also permits .metadata files for source distributions,
|
||||
# but PyPI doesn't serve such files. In addition, an sdist's metadata
|
||||
# is generated at build time and may legitimately differ from what the
|
||||
# index declared, so it's been decided to skip this check for sdists.
|
||||
# This can change later if needed.
|
||||
#
|
||||
# TODO: this is a hack for checking whether a distribution is metadata-
|
||||
# only or not. If/when we refactor distributions to delineate between
|
||||
# metadata-only and concrete distributions, clean this up.
|
||||
if (
|
||||
link.is_wheel
|
||||
and req._distribution is not None
|
||||
and req._distribution is not dist
|
||||
and link.metadata_link() is not None
|
||||
):
|
||||
_check_sidecar_matches_wheel(req, req._distribution, dist)
|
||||
|
||||
return dist
|
||||
|
||||
def save_linked_requirement(self, req: InstallRequirement) -> None:
|
||||
@@ -683,7 +819,7 @@ class RequirementPreparer:
|
||||
# No distribution was downloaded for this requirement.
|
||||
return
|
||||
|
||||
download_location = os.path.join(self.download_dir, link.filename)
|
||||
download_location = join_within_directory(self.download_dir, link.filename)
|
||||
if not os.path.exists(download_location):
|
||||
shutil.copy(req.local_file_path, download_location)
|
||||
download_path = display_path(download_location)
|
||||
@@ -716,6 +852,7 @@ class RequirementPreparer:
|
||||
self.build_env_installer,
|
||||
self.build_isolation,
|
||||
self.check_build_deps,
|
||||
self.allow_editables,
|
||||
)
|
||||
|
||||
req.check_if_exists(self.use_user_site)
|
||||
|
||||
Reference in new issue
Block a user