This commit is contained in:
wittmannzl103469 committed 2026-11-05 15:08:42 +01:00
1 parent de41a02749
commit 9af09cebd3
764 files changed
+54218 -78859

No files matched your search

@@ -2,12 +2,13 @@
distributions."""
import importlib.resources
import locale
import logging
import os
import sys
from typing import IO
__all__ = ["get_path_uid", "stdlib_pkgs", "tomllib", "WINDOWS"]
__all__ = ["get_locale_encoding", "get_path_uid", "tomllib", "WINDOWS"]
logger = logging.getLogger(__name__)
@@ -26,6 +27,19 @@ def has_tls() -> bool:
return IS_PYOPENSSL
def get_locale_encoding() -> str:
"""Return the locale encoding.
Uses ``locale.getencoding()`` when available (Python 3.11+) to avoid the
``EncodingWarning`` that ``locale.getpreferredencoding(False)`` raises
under UTF-8 Mode.
"""
# TODO: Remove the 3.10 fallback once pip drops Python 3.10 support.
if sys.version_info >= (3, 11):
return locale.getencoding()
return locale.getpreferredencoding(False)
def get_path_uid(path: str) -> int:
"""
Return path's uid.
@@ -73,13 +87,5 @@ else:
from pip._vendor import tomli as tomllib
# packages in the stdlib that may have installation metadata, but should not be
# considered 'installed'. this theoretically could be determined based on
# dist.location (py27:`sysconfig.get_paths()['stdlib']`,
# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may
# make this ineffective, so hard-coding
stdlib_pkgs = {"python", "wsgiref", "argparse"}
# windows detection, covers cpython and ironpython
WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
@@ -5,10 +5,16 @@ A module that implements tooling to enable easy warnings about deprecations.
from __future__ import annotations
import logging
import os
import warnings
from collections.abc import Generator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any, TextIO
from pip._vendor.packaging.version import parse
from pip._vendor.pyproject_hooks import BuildBackendWarning
from pip._vendor.rich.text import Text
from pip import __version__ as current_version # NOTE: tests patch this name.
@@ -16,10 +22,23 @@ DEPRECATION_MSG_PREFIX = "DEPRECATION: "
class PipDeprecationWarning(Warning):
pass
include_source: bool = False
_original_showwarning: Any = None
_build_backend_warning_level = ContextVar(
"build_backend_warning_level", default=logging.DEBUG
)
@contextmanager
def build_backend_warnings(level: int) -> Generator[None, None, None]:
"""Set the logging level for warnings from this build backend call."""
token = _build_backend_warning_level.set(level)
try:
yield
finally:
_build_backend_warning_level.reset(token)
# Warnings <-> Logging Integration
@@ -38,14 +57,30 @@ def _showwarning(
# We use a specially named logger which will handle all of the
# deprecation messages for pip.
logger = logging.getLogger("pip._internal.deprecations")
logger.warning(message)
if isinstance(message, PipDeprecationWarning) and message.include_source:
logger.warning("%s (%s:%s)", message, filename, lineno)
else:
logger.warning(message)
elif issubclass(category, BuildBackendWarning):
logger = logging.getLogger("pip._internal.build_backend")
formatted = warnings.formatwarning(message, category, filename, lineno, line)
logger.log(
_build_backend_warning_level.get(),
"%s",
Text(formatted.rstrip()),
extra={"rich": True},
)
else:
_original_showwarning(message, category, filename, lineno, file, line)
def install_warning_logger() -> None:
# Enable our Deprecation Warnings
warnings.simplefilter("default", PipDeprecationWarning, append=True)
# If we're running pip test suite, promote the PipDeprecationWarning into errors.
if os.environ.get("_PIP_TEST_ENV", None):
warnings.simplefilter("error", PipDeprecationWarning)
else:
warnings.simplefilter("default", PipDeprecationWarning, append=True)
global _original_showwarning
@@ -61,6 +96,8 @@ def deprecated(
gone_in: str | None,
feature_flag: str | None = None,
issue: int | None = None,
stacklevel: int = 2,
include_source: bool = False,
) -> None:
"""Helper to deprecate existing functionality.
@@ -80,6 +117,12 @@ def deprecated(
issue:
Issue number on the tracker that would serve as a useful place for
users to find related discussion and provide feedback.
stacklevel:
How many frames up the call stack to attribute the warning to.
Defaults to 2 (the caller of deprecated()).
include_source:
If True, include the source filename and line number in the warning
output. Useful when the warning originates from external code.
"""
# Determine whether or not the feature is already gone in this version.
@@ -123,4 +166,6 @@ def deprecated(
if is_gone:
raise PipDeprecationWarning(message)
warnings.warn(message, category=PipDeprecationWarning, stacklevel=2)
warning = PipDeprecationWarning(message)
warning.include_source = include_source
warnings.warn(warning, stacklevel=stacklevel)
@@ -11,16 +11,20 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s
direct_url.validate() # if invalid, this is a pip bug
requirement = name + " @ "
fragments = []
if isinstance(direct_url.info, VcsInfo):
if direct_url.vcs_info:
requirement += (
f"{direct_url.info.vcs}+{direct_url.url}@{direct_url.info.commit_id}"
f"{direct_url.vcs_info.vcs}+{direct_url.url}"
f"@{direct_url.vcs_info.commit_id}"
)
elif isinstance(direct_url.info, ArchiveInfo):
elif direct_url.archive_info:
requirement += direct_url.url
if direct_url.info.hash:
fragments.append(direct_url.info.hash)
if direct_url.archive_info.hashes:
hash_algorithm, hash_value = next(
iter(direct_url.archive_info.hashes.items())
)
fragments.append(f"{hash_algorithm}={hash_value}")
else:
assert isinstance(direct_url.info, DirInfo)
assert direct_url.dir_info
requirement += direct_url.url
if direct_url.subdirectory:
fragments.append("subdirectory=" + direct_url.subdirectory)
@@ -32,7 +36,7 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s
def direct_url_for_editable(source_dir: str) -> DirectUrl:
return DirectUrl(
url=path_to_url(source_dir),
info=DirInfo(editable=True),
dir_info=DirInfo(editable=True),
)
@@ -62,7 +66,7 @@ def direct_url_from_link(
commit_id = vcs_backend.get_revision(source_dir)
return DirectUrl(
url=url,
info=VcsInfo(
vcs_info=VcsInfo(
vcs=vcs_backend.name,
commit_id=commit_id,
requested_revision=requested_revision,
@@ -72,16 +76,17 @@ def direct_url_from_link(
elif link.is_existing_dir():
return DirectUrl(
url=link.url_without_fragment,
info=DirInfo(),
dir_info=DirInfo(),
subdirectory=link.subdirectory_fragment,
)
else:
hash = None
hash_name = link.hash_name
if hash_name:
hash = f"{hash_name}={link.hash}"
if link.hash_name:
assert link.hash
hashes = {link.hash_name: link.hash}
else:
hashes = None
return DirectUrl(
url=link.url_without_fragment,
info=ArchiveInfo(hash=hash),
archive_info=ArchiveInfo(hashes=hashes),
subdirectory=link.subdirectory_fragment,
)
@@ -5,11 +5,11 @@ import os
import os.path
import random
import sys
from collections.abc import Generator
from collections.abc import Callable, Generator
from contextlib import contextmanager
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, BinaryIO, Callable, cast
from typing import Any, BinaryIO, cast
from pip._internal.utils.compat import get_path_uid
from pip._internal.utils.misc import format_size
@@ -172,7 +172,7 @@ def _subdirs_without_generic(
predicate under it."""
directories = []
excluded = set()
excluded: set[Path] = set()
for root_str, _, filenames in os.walk(Path(path).resolve()):
root = Path(root_str)
@@ -180,9 +180,7 @@ def _subdirs_without_generic(
# This directory should be excluded, so exclude it and all of its
# parent directories.
# The last item in root.parents is ".", so we ignore it.
#
# Wrapping this in `list()` is only needed for Python 3.9.
excluded.update(list(root.parents)[:-1])
excluded.update(root.parents[:-1])
excluded.add(root)
directories.append(root)
@@ -8,9 +8,9 @@ from pip._internal.exceptions import HashMismatch, HashMissing, InstallationErro
from pip._internal.utils.misc import read_chunks
if TYPE_CHECKING:
from collections.abc import Mapping
from hashlib import _Hash
# The recommended hash algo of the moment. Change this whenever the state of
# the art changes; it won't hurt backward compatibility.
FAVORITE_HASH = "sha256"
@@ -104,7 +104,7 @@ class Hashes:
with open(path, "rb") as file:
return self.check_against_file(file)
def has_one_of(self, hashes: dict[str, str]) -> bool:
def has_one_of(self, hashes: Mapping[str, str]) -> bool:
"""Return whether any of the given hashes are allowed."""
for hash_name, hex_digest in hashes.items():
if self.is_hash_allowed(hash_name, hex_digest):
@@ -29,7 +29,7 @@ from pip._vendor.rich.style import Style
from pip._internal.utils._log import VERBOSE, getLogger
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
from pip._internal.utils.misc import StreamWrapper, ensure_dir
from pip._internal.utils.misc import StreamWrapper, ensure_dir, looks_like_ci
_log_state = threading.local()
_stdout_console = None
@@ -320,8 +320,16 @@ def setup_logging(verbosity: int, no_color: bool, user_log_file: str | None) ->
["user_log"] if include_user_log else []
)
global _stdout_console, stderr_console
_stdout_console = PipConsole(file=sys.stdout, no_color=no_color, soft_wrap=True)
_stderr_console = PipConsole(file=sys.stderr, no_color=no_color, soft_wrap=True)
console_options: dict[str, Any] = {"no_color": no_color, "soft_wrap": True}
if looks_like_ci():
# Don't animate progress bars and status spinners when running
# in CI. A lot of CI workflows use FORCE_COLOR and similar envvars
# so their CI output remains colorized, but this causes rich to
# assume a fully interactive terminal, resulting in any animated
# output to span many lines which is distracting (and unhelpful).
console_options["force_interactive"] = False
_stdout_console = PipConsole(file=sys.stdout, **console_options)
_stderr_console = PipConsole(file=sys.stderr, **console_options)
logging.config.dictConfig(
{
@@ -5,15 +5,16 @@ import getpass
import hashlib
import logging
import os
import pathlib
import posixpath
import shutil
import stat
import sys
import sysconfig
import urllib.parse
from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from functools import cache, partial, wraps
from io import StringIO
from itertools import filterfalse, tee, zip_longest
from pathlib import Path
@@ -21,20 +22,25 @@ from types import FunctionType, TracebackType
from typing import (
Any,
BinaryIO,
Callable,
Optional,
NoReturn,
TextIO,
TypeVar,
cast,
)
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pip._vendor.pyproject_hooks import BackendUnavailable, BuildBackendHookCaller
from pip import __file__ as pip_location
from pip import __version__
from pip._internal.exceptions import CommandError, ExternallyManagedEnvironment
from pip._internal.exceptions import (
BackendUnavailableError,
CommandError,
ExternallyManagedEnvironment,
)
from pip._internal.locations import get_major_minor_version
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.deprecation import build_backend_warnings
from pip._internal.utils.retry import retry
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -52,19 +58,38 @@ __all__ = [
"ensure_dir",
"remove_auth_from_url",
"check_externally_managed",
"looks_like_ci",
"ConfiguredBuildBackendHookCaller",
]
logger = logging.getLogger(__name__)
T = TypeVar("T")
F = TypeVar("F", bound=Callable[..., Any])
ExcInfo = tuple[type[BaseException], BaseException, TracebackType]
VersionInfo = tuple[int, int, int]
NetlocTuple = tuple[str, tuple[Optional[str], Optional[str]]]
NetlocTuple = tuple[str, tuple[str | None, str | None]]
OnExc = Callable[[FunctionType, Path, BaseException], Any]
OnErr = Callable[[FunctionType, Path, ExcInfo], Any]
FILE_CHUNK_SIZE = 1024 * 1024
# These are environment variables present when running under various
# CI systems. For each variable, some CI systems that use the variable
# are indicated. The collection was chosen so that for each of a number
# of popular systems, at least one of the environment variables is used.
# This list is used to provide some indication of and lower bound for
# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive.
# For more background, see: https://github.com/pypa/pip/issues/5499
CI_ENVIRONMENT_VARIABLES = (
# Azure Pipelines
"BUILD_BUILDID",
# Jenkins
"BUILD_ID",
# AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI
"CI",
# Explicit environment variable.
"PIP_IS_CI",
)
def get_pip_version() -> str:
@@ -74,6 +99,27 @@ def get_pip_version() -> str:
return f"pip {__version__} from {pip_pkg_dir} (python {get_major_minor_version()})"
@cache
def running_under_zipapp() -> bool:
return not pathlib.Path(pip_location).resolve().parent.is_dir()
def get_runnable_pip() -> str:
"""Get a file to pass to a Python executable, to run the currently-running pip.
This is used to run a pip subprocess, for installing requirements into the build
environment.
"""
source = pathlib.Path(pip_location).resolve().parent
if running_under_zipapp():
# This would happen if someone is using pip from inside a zip file. In that
# case, we can use that directly.
return str(source)
return os.fsdecode(source / "__pip-runner__.py")
def normalize_version_info(py_version_info: tuple[int, ...]) -> tuple[int, int, int]:
"""
Convert a tuple of ints representing a Python version to one of length
@@ -385,14 +431,6 @@ class StreamWrapper(StringIO):
return self.orig_stream.encoding
# Simulates an enum
def enum(*sequential: Any, **named: Any) -> type[Any]:
enums = dict(zip(sequential, range(len(sequential))), **named)
reverse = {value: key for key, value in enums.items()}
enums["reverse_mapping"] = reverse
return type("Enum", (), enums)
def build_netloc(host: str, port: int | None) -> str:
"""
Build a netloc from a host-port pair
@@ -548,7 +586,7 @@ class HiddenText:
if type(self) is type(other):
# The string being used for redaction doesn't also have to match,
# just the raw, original string.
return self.secret == cast(HiddenText, other).secret
return self.secret == other.secret
return NotImplemented
# Disable hashing, since we have a custom __eq__ and don't need hash-ability
@@ -648,6 +686,24 @@ def partition(
return filterfalse(pred, t1), filter(pred, t2)
def _build_backend_hook(func: F) -> F:
"""Apply warning policy and translate failures for a public backend hook."""
@wraps(func)
def wrapper(
self: ConfiguredBuildBackendHookCaller,
*args: Any,
**kwargs: Any,
) -> Any:
with build_backend_warnings(self.backend_warning_level):
try:
return func(self, *args, **kwargs)
except BackendUnavailable as error:
self._raise_backend_unavailable(func.__name__, error)
return cast(F, wrapper)
class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
def __init__(
self,
@@ -657,12 +713,34 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
backend_path: str | None = None,
runner: Callable[..., None] | None = None,
python_executable: str | None = None,
*,
backend_warning_level: int = logging.DEBUG,
):
super().__init__(
source_dir, build_backend, backend_path, runner, python_executable
)
self.config_holder = config_holder
self.backend_warning_level = backend_warning_level
def _raise_backend_unavailable(
self, hook_name: str, error: BackendUnavailable
) -> NoReturn:
backend_error = (
error.traceback.strip().splitlines()[-1] if error.traceback else str(error)
)
exception = BackendUnavailableError(
hook_name=hook_name,
backend_name=self.build_backend,
backend_error=backend_error,
)
logger.error("%s", exception, extra={"rich": True})
raise exception from error
@_build_backend_hook
def supports_feature(self, feature: str) -> bool:
return feature in self._supported_features()
@_build_backend_hook
def build_wheel(
self,
wheel_directory: str,
@@ -674,6 +752,7 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
wheel_directory, config_settings=cs, metadata_directory=metadata_directory
)
@_build_backend_hook
def build_sdist(
self,
sdist_directory: str,
@@ -682,6 +761,7 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
cs = self.config_holder.config_settings
return super().build_sdist(sdist_directory, config_settings=cs)
@_build_backend_hook
def build_editable(
self,
wheel_directory: str,
@@ -693,24 +773,28 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
wheel_directory, config_settings=cs, metadata_directory=metadata_directory
)
@_build_backend_hook
def get_requires_for_build_wheel(
self, config_settings: Mapping[str, Any] | None = None
) -> Sequence[str]:
cs = self.config_holder.config_settings
return super().get_requires_for_build_wheel(config_settings=cs)
@_build_backend_hook
def get_requires_for_build_sdist(
self, config_settings: Mapping[str, Any] | None = None
) -> Sequence[str]:
cs = self.config_holder.config_settings
return super().get_requires_for_build_sdist(config_settings=cs)
@_build_backend_hook
def get_requires_for_build_editable(
self, config_settings: Mapping[str, Any] | None = None
) -> Sequence[str]:
cs = self.config_holder.config_settings
return super().get_requires_for_build_editable(config_settings=cs)
@_build_backend_hook
def prepare_metadata_for_build_wheel(
self,
metadata_directory: str,
@@ -724,12 +808,13 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
_allow_fallback=_allow_fallback,
)
@_build_backend_hook
def prepare_metadata_for_build_editable(
self,
metadata_directory: str,
config_settings: Mapping[str, Any] | None = None,
_allow_fallback: bool = True,
) -> str | None:
) -> str:
cs = self.config_holder.config_settings
return super().prepare_metadata_for_build_editable(
metadata_directory=metadata_directory,
@@ -769,3 +854,13 @@ def warn_if_run_as_root() -> None:
"Use the --root-user-action option if you know what you are doing and "
"want to suppress this warning."
)
def looks_like_ci() -> bool:
"""
Return whether it looks like pip is running under CI.
"""
# We don't use the method of checking for a tty (e.g. using isatty())
# because some CI systems mimic a tty (e.g. Travis CI). Thus that
# method doesn't provide definitive information in either direction.
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
@@ -1,5 +1,6 @@
from __future__ import annotations
import copy
import functools
import logging
@@ -42,3 +43,14 @@ def get_requirement(req_string: str) -> Requirement:
# minimize repeated parsing of the same string to construct equivalent
# Requirement objects.
return Requirement(req_string)
def requirement_without_marker(req: Requirement) -> Requirement:
"""Return a copy of ``req`` with its environment marker removed.
Intended for error messages. The original is left unchanged, which
matters because ``get_requirement`` caches and shares instances.
"""
stripped = copy.copy(req)
stripped.marker = None
return stripped
@@ -1,5 +1,10 @@
from collections.abc import Iterable
from __future__ import annotations
import os
from collections.abc import Collection, Iterable, Iterator
from pathlib import Path
from typing import TYPE_CHECKING
from urllib.parse import urljoin, urlsplit
from pip._vendor.packaging.pylock import (
Package,
@@ -9,13 +14,20 @@ from pip._vendor.packaging.pylock import (
PackageVcs,
PackageWheel,
Pylock,
is_valid_pylock_path,
)
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo
from pip._internal.exceptions import DiagnosticPipError, InstallationError
from pip._internal.models.link import Link
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.urls import url_to_path
from pip._internal.utils.compat import tomllib
from pip._internal.utils.urls import path_to_url, url_to_path
if TYPE_CHECKING:
from pip._internal.index.package_finder import FormatControl
from pip._internal.network.session import PipSession
from pip._internal.req.req_install import InstallRequirement
def _pylock_package_from_install_requirement(
@@ -32,16 +44,16 @@ def _pylock_package_from_install_requirement(
package_sdist = None
package_wheels = None
if ireq.is_direct:
if isinstance(download_info.info, VcsInfo):
if download_info.vcs_info:
package_vcs = PackageVcs(
type=download_info.info.vcs,
type=download_info.vcs_info.vcs,
url=download_info.url,
path=None,
requested_revision=download_info.info.requested_revision,
commit_id=download_info.info.commit_id,
requested_revision=download_info.vcs_info.requested_revision,
commit_id=download_info.vcs_info.commit_id,
subdirectory=download_info.subdirectory,
)
elif isinstance(download_info.info, DirInfo):
elif download_info.dir_info:
package_directory = PackageDirectory(
path=(
Path(url_to_path(download_info.url))
@@ -50,17 +62,19 @@ def _pylock_package_from_install_requirement(
.as_posix()
),
editable=(
download_info.info.editable if download_info.info.editable else None
download_info.dir_info.editable
if download_info.dir_info.editable
else None
),
subdirectory=download_info.subdirectory,
)
elif isinstance(download_info.info, ArchiveInfo):
if not download_info.info.hashes:
elif download_info.archive_info:
if not download_info.archive_info.hashes:
raise NotImplementedError()
package_archive = PackageArchive(
url=download_info.url,
path=None,
hashes=download_info.info.hashes,
hashes=download_info.archive_info.hashes,
subdirectory=download_info.subdirectory,
)
else:
@@ -68,8 +82,8 @@ def _pylock_package_from_install_requirement(
raise NotImplementedError()
else:
package_version = dist.version
if isinstance(download_info.info, ArchiveInfo):
if not download_info.info.hashes:
if download_info.archive_info:
if not download_info.archive_info.hashes:
raise NotImplementedError()
link = Link(download_info.url)
if link.is_wheel:
@@ -77,14 +91,14 @@ def _pylock_package_from_install_requirement(
PackageWheel(
name=link.filename,
url=download_info.url,
hashes=download_info.info.hashes,
hashes=download_info.archive_info.hashes,
)
]
else:
package_sdist = PackageSdist(
name=link.filename,
url=download_info.url,
hashes=download_info.info.hashes,
hashes=download_info.archive_info.hashes,
)
else:
# should never happen
@@ -114,3 +128,197 @@ def pylock_from_install_requirements(
key=lambda p: p.name,
),
)
def _is_url(s: str) -> bool:
return s.lower().startswith(("http:", "https:", "file:"))
def is_valid_pylock_filename(filename: str, *, allow_urls: bool = True) -> bool:
"""Check a lock source's filename, optionally restricting it to local paths."""
if _is_url(filename):
if not allow_urls:
return False
try:
path = Path(urlsplit(filename).path.rpartition("/")[-1])
except ValueError:
return False
else:
path = Path(filename)
return is_valid_pylock_path(path)
def _package_dist_url(
pylock_path_or_url: str, path: str | None, url: str | None
) -> str:
"""Compute an url from a Pylock package path and url.
Give priority to path over url. If path is relative,
compute an url using the pylock file location as base.
"""
if path is not None:
if not os.path.isabs(path):
# relative path, join to pylock location
if _is_url(pylock_path_or_url):
base = urlsplit(pylock_path_or_url)
dist_url = urljoin(base.geturl(), path)
# os.path.isabs does not treat a scheme-carrying value like
# "file:..." as absolute, so it reaches here and urljoin honors
# its scheme, discarding the pylock base. Only keep the result
# if its scheme and host still match the lock's own.
target = urlsplit(dist_url)
if (target.scheme, target.netloc) != (base.scheme, base.netloc):
raise InstallationError(
f"Path {path!r} in pylock file obtained from a URL "
f"resolves outside its location: {pylock_path_or_url!r}"
)
return dist_url
else:
return path_to_url(
os.path.join(os.path.dirname(pylock_path_or_url), path)
)
else:
# absolute path, reject if pylock comes from a URL
if _is_url(pylock_path_or_url):
raise InstallationError(
f"Absolute paths are not supported in pylock files obtained "
f"from a URL: {path!r} in {pylock_path_or_url!r}"
)
return path_to_url(path)
else:
assert url is not None # guaranteed by packaging.pylock validation
return url
def package_vcs_requirement_url(
pylock_path_or_url: str, package_vcs: PackageVcs
) -> str:
dist_url = _package_dist_url(pylock_path_or_url, package_vcs.path, package_vcs.url)
url = f"{package_vcs.type}+{dist_url}@{package_vcs.commit_id}"
if package_vcs.subdirectory:
if "#" in url:
raise InstallationError(
f"Package URL {url!r} cannot contain fragments in combination "
f"with subdirectory field (in {pylock_path_or_url!r})"
)
url += "#subdirectory=" + package_vcs.subdirectory
return url
def package_archive_requirement_url(
pylock_path_or_url: str, package_archive: PackageArchive
) -> str:
url = _package_dist_url(
pylock_path_or_url, package_archive.path, package_archive.url
)
if package_archive.subdirectory:
if "#" in url:
raise InstallationError(
f"Package URL {url!r} cannot contain fragments in combination "
f"with subdirectory field (in {pylock_path_or_url!r})"
)
url += "#subdirectory=" + package_archive.subdirectory
return url
def package_directory_requirement_url(
pylock_path_or_url: str, package_directory: PackageDirectory
) -> str:
if _is_url(pylock_path_or_url) and not pylock_path_or_url.lower().startswith(
"file:"
):
raise InstallationError(
f"Directory entries are not supported in remote pylock.toml "
f"{pylock_path_or_url!r}"
)
url = _package_dist_url(pylock_path_or_url, package_directory.path, None)
assert url.startswith("file://")
if not url.endswith("/"):
url += "/"
if package_directory.subdirectory:
url += package_directory.subdirectory
if not url.endswith("/"):
url += "/"
return url
def package_sdist_requirement_url(
pylock_path_or_url: str, package_sdist: PackageSdist
) -> str:
return _package_dist_url(pylock_path_or_url, package_sdist.path, package_sdist.url)
def package_wheel_requirement_url(
pylock_path_or_url: str, package_wheel: PackageWheel
) -> str:
return _package_dist_url(pylock_path_or_url, package_wheel.path, package_wheel.url)
def _get_pylock_path_or_url_content(path_or_url: str, session: PipSession) -> str:
# TODO: refactor - this is similar to req_file.get_file_content
# Pip has special support for file:// URLs (LocalFSAdapter).
if _is_url(path_or_url):
# Delay importing heavy network modules until absolutely necessary.
from pip._internal.network.utils import raise_for_status
resp = session.get(path_or_url)
raise_for_status(resp)
return resp.text
# Assume this is a bare path.
return Path(path_or_url).read_text(encoding="utf-8")
def select_from_pylock_path_or_url(
pylock_path_or_url: str,
session: PipSession,
format_control: FormatControl | None,
*,
extras: Collection[str] | None = None,
dependency_groups: Iterable[str] = (),
include_default_groups: bool = True,
) -> Iterator[
tuple[
Package,
PackageVcs | PackageDirectory | PackageArchive | PackageWheel | PackageSdist,
]
]:
try:
pylock_content = _get_pylock_path_or_url_content(pylock_path_or_url, session)
except DiagnosticPipError:
raise
except Exception as exc:
raise InstallationError(
f"Error reading pylock file {pylock_path_or_url!r}: {exc}"
) from exc
try:
lock = Pylock.from_dict(tomllib.loads(pylock_content))
except Exception as exc:
raise InstallationError(
f"Invalid pylock file {pylock_path_or_url!r}: {exc}"
) from exc
selected_groups: set[str] = {canonicalize_name(name) for name in dependency_groups}
if include_default_groups:
selected_groups.update(lock.default_groups or ())
def prefer_sdist_predicate(name: NormalizedName) -> bool:
if format_control is None:
return False
allowed_formats = format_control.get_allowed_formats(name)
if "source" in allowed_formats and "binary" not in allowed_formats:
return True
return False
try:
# packaging 26.4 will reject undeclared extras and dependency groups.
yield from lock.select(
extras=extras,
dependency_groups=selected_groups,
prefer_sdist_predicate=prefer_sdist_predicate,
)
except Exception as exc:
raise InstallationError(
f"Cannot select requirements from pylock file {pylock_path_or_url!r}: {exc}"
) from exc
@@ -1,8 +1,9 @@
from __future__ import annotations
import functools
from collections.abc import Callable
from time import perf_counter, sleep
from typing import TYPE_CHECKING, Callable, TypeVar
from typing import TYPE_CHECKING, TypeVar
if TYPE_CHECKING:
from typing_extensions import ParamSpec
@@ -4,8 +4,8 @@ import logging
import os
import shlex
import subprocess
from collections.abc import Iterable, Mapping
from typing import Any, Callable, Literal, Union
from collections.abc import Callable, Iterable, Mapping
from typing import Any, Literal
from pip._vendor.rich.markup import escape
@@ -14,7 +14,7 @@ from pip._internal.exceptions import InstallationSubprocessError
from pip._internal.utils.logging import VERBOSE, subprocess_logger
from pip._internal.utils.misc import HiddenText
CommandArgs = list[Union[str, HiddenText]]
CommandArgs = list[str | HiddenText]
def make_command(*args: str | HiddenText | CommandArgs) -> CommandArgs:
@@ -134,6 +134,7 @@ def call_subprocess(
stderr=subprocess.STDOUT if not stdout_only else subprocess.PIPE,
cwd=cwd,
env=env,
encoding="locale",
errors="backslashreplace",
)
except Exception as exc:
@@ -6,16 +6,16 @@ import logging
import os.path
import tempfile
import traceback
from collections.abc import Generator
from collections.abc import Callable, Generator
from contextlib import ExitStack, contextmanager
from enum import Enum
from pathlib import Path
from typing import (
Any,
Callable,
TypeVar,
)
from pip._internal.utils.misc import enum, rmtree
from pip._internal.utils.misc import rmtree
logger = logging.getLogger(__name__)
@@ -24,11 +24,13 @@ _T = TypeVar("_T", bound="TempDirectory")
# Kinds of temporary directories. Only needed for ones that are
# globally-managed.
tempdir_kinds = enum(
BUILD_ENV="build-env",
EPHEM_WHEEL_CACHE="ephem-wheel-cache",
REQ_BUILD="req-build",
)
# TODO: Use StrEnum when Python 3.10 support is dropped.
class TempDirectoryKind(str, Enum):
BUILD_ENV = "build-env"
EPHEM_WHEEL_CACHE = "ephem-wheel-cache"
REQ_BUILD = "req-build"
__str__ = str.__str__
_tempdir_manager: ExitStack | None = None
@@ -76,15 +76,26 @@ def has_leading_dir(paths: Iterable[str]) -> bool:
return True
def is_within_directory(directory: str, target: str) -> bool:
def is_within_directory(
directory: str, target: str, *, resolve_symlinks: bool = False
) -> bool:
"""
Return true if the absolute path of target is within the directory
"""
abs_directory = os.path.abspath(directory)
abs_target = os.path.abspath(target)
(including when target is equal to the directory).
prefix = os.path.commonpath([abs_directory, abs_target])
return prefix == abs_directory
When ``resolve_symlinks`` is true, resolve symlinks before comparing so
traversal through a symlink (e.g. "link/../file") is also caught.
"""
if resolve_symlinks:
abs_directory = os.path.realpath(directory)
abs_target = os.path.realpath(target)
else:
abs_directory = os.path.abspath(directory)
abs_target = os.path.abspath(target)
return abs_target == abs_directory or abs_target.startswith(
os.path.join(abs_directory, "")
)
def _get_default_mode_plus_executable() -> int:
@@ -277,7 +288,13 @@ def _untar_without_filter(
if leading:
fn = split_leading_dir(fn)[1]
path = os.path.join(location, fn)
if not is_within_directory(location, path):
# The plain check rejects textual ".." escapes; resolving symlinks also
# catches a later member redirected outside by an earlier member's
# symlink (e.g. "link/../file").
if not is_within_directory(location, path) or not is_within_directory(
location, path, resolve_symlinks=True
):
message = (
"The tar file ({}) has a file ({}) trying to install "
"outside target directory ({})"
@@ -286,6 +303,17 @@ def _untar_without_filter(
if member.isdir():
ensure_dir(path)
elif member.issym():
# Reject symlinks resolving outside the destination, so a later
# member cannot be written through them.
target = os.path.join(os.path.dirname(path), member.linkname)
if not is_within_directory(location, target, resolve_symlinks=True):
message = (
"The tar file ({}) has a file ({}) trying to install "
"outside target directory ({})"
)
raise InstallationError(
message.format(filename, member.name, member.linkname)
)
if not is_symlink_target_in_tar(tar, member):
message = (
"The tar file ({}) has a file ({}) trying to install "
@@ -336,27 +364,46 @@ def unpack_file(
location: str,
content_type: str | None = None,
) -> None:
"""Unpack ``filename`` into ``location``.
Archive format is chosen in order of decreasing reliability:
``content_type``, then filename extension, then magic signature
(unambiguous matches only).
"""
filename = os.path.realpath(filename)
if (
content_type == "application/zip"
or filename.lower().endswith(ZIP_EXTENSIONS)
or zipfile.is_zipfile(filename)
):
unzip_file(filename, location, flatten=not filename.endswith(".whl"))
elif (
content_type == "application/x-gzip"
or tarfile.is_tarfile(filename)
or filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)
):
zip_flatten = not filename.endswith(".whl")
def _unzip() -> None:
unzip_file(filename, location, flatten=zip_flatten)
def _untar() -> None:
untar_file(filename, location)
else:
# FIXME: handle?
# FIXME: magic signatures?
logger.critical(
"Cannot unpack file %s (downloaded from %s, content-type: %s); "
"cannot detect archive format",
filename,
location,
content_type,
)
raise InstallationError(f"Cannot determine archive format of {location}")
if content_type == "application/zip":
return _unzip()
if content_type == "application/x-gzip":
return _untar()
if filename.lower().endswith(ZIP_EXTENSIONS):
return _unzip()
if filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS):
return _untar()
# avoid ambiguous case where both signature checks return True
is_zipfile = zipfile.is_zipfile(filename)
is_tarfile = tarfile.is_tarfile(filename)
if is_zipfile and not is_tarfile:
return _unzip()
if is_tarfile and not is_zipfile:
return _untar()
if is_zipfile and is_tarfile:
logger.error("Ambiguous file signature in %s.", filename)
logger.critical(
"Cannot unpack file %s (downloaded from %s, content-type: %s); "
"cannot detect archive format",
filename,
location,
content_type,
)
raise InstallationError(f"Cannot determine archive format of {location}")
@@ -1,9 +1,10 @@
import os
import re
import string
import urllib.parse
import urllib.request
import urllib.parse # noqa: F401
from .compat import WINDOWS
from .packaging import get_requirement
def path_to_url(path: str) -> str:
@@ -11,6 +12,8 @@ def path_to_url(path: str) -> str:
Convert a path to a file: URL. The path will be made absolute and have
quoted path parts.
"""
import urllib.request
path = os.path.normpath(os.path.abspath(path))
url = urllib.parse.urljoin("file://", urllib.request.pathname2url(path))
return url
@@ -20,6 +23,8 @@ def url_to_path(url: str) -> str:
"""
Convert a file: URL to a path.
"""
import urllib.request
assert url.startswith(
"file:"
), f"You can only turn file: urls into filenames (not {url!r})"
@@ -53,3 +58,21 @@ def url_to_path(url: str) -> str:
path = path[1:]
return path
def _convert_extras(extras: str | None) -> set[str]:
if not extras:
return set()
return get_requirement("placeholder" + extras.lower()).extras
def strip_extras(path: str) -> tuple[str, set[str]]:
m = re.match(r"^(.+)(\[[^\]]+\])$", path)
extras = None
if m:
path_no_extras = m.group(1).rstrip()
extras = m.group(2)
else:
path_no_extras = path
return path_no_extras, _convert_extras(extras)
@@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import os
import re
import site
import sys
logger = logging.getLogger(__name__)
@@ -12,7 +11,7 @@ _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile(
)
def _running_under_venv() -> bool:
def running_under_virtualenv() -> bool:
"""Checks if sys.base_prefix and sys.prefix match.
This handles PEP 405 compliant virtual environments.
@@ -20,20 +19,6 @@ def _running_under_venv() -> bool:
return sys.prefix != getattr(sys, "base_prefix", sys.prefix)
def _running_under_legacy_virtualenv() -> bool:
"""Checks if sys.real_prefix is set.
This handles virtual environments created with pypa's virtualenv.
"""
# pypa/virtualenv case
return hasattr(sys, "real_prefix")
def running_under_virtualenv() -> bool:
"""True if we're running inside a virtual environment, False otherwise."""
return _running_under_venv() or _running_under_legacy_virtualenv()
def _get_pyvenv_cfg_lines() -> list[str] | None:
"""Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines
@@ -78,28 +63,9 @@ def _no_global_under_venv() -> bool:
return False
def _no_global_under_legacy_virtualenv() -> bool:
"""Check if "no-global-site-packages.txt" exists beside site.py
This mirrors logic in pypa/virtualenv for determining whether system
site-packages are visible in the virtual environment.
"""
site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
no_global_site_packages_file = os.path.join(
site_mod_dir,
"no-global-site-packages.txt",
)
return os.path.exists(no_global_site_packages_file)
def virtualenv_no_global() -> bool:
"""Returns a boolean, whether running in venv with no system site-packages."""
# PEP 405 compliance needs to be checked first since virtualenv >=20 would
# return True for both checks, but is only able to use the PEP 405 config.
if _running_under_venv():
if running_under_virtualenv():
return _no_global_under_venv()
if _running_under_legacy_virtualenv():
return _no_global_under_legacy_virtualenv()
return False