This commit is contained in:
wittmannzl103469 committed 2026-11-05 15:08:42 +01:00
1 parent de41a02749
commit 9af09cebd3
764 files changed
+54218 -78859

No files matched your search

@@ -44,6 +44,7 @@ def install_given_reqs(
use_user_site: bool,
pycompile: bool,
progress_bar: BarType,
script_executable: str | None = None,
) -> list[InstallationResult]:
"""
Install everything in the given list.
@@ -73,6 +74,7 @@ def install_given_reqs(
for requirement in items:
req_name = requirement.name
assert req_name is not None
if requirement.should_reinstall:
logger.info("Attempting uninstall: %s", req_name)
with indent_log():
@@ -88,6 +90,10 @@ def install_given_reqs(
warn_script_location=warn_script_location,
use_user_site=use_user_site,
pycompile=pycompile,
script_executable=script_executable,
rollback_on_error=bool(
uninstalled_pathset and uninstalled_pathset.can_rollback
),
)
except Exception:
# if install did not succeed, rollback previous uninstall
@@ -14,15 +14,15 @@ import copy
import logging
import os
import re
from collections.abc import Collection
from collections.abc import Collection, Mapping
from dataclasses import dataclass
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging import pylock
from pip._vendor.packaging.markers import InvalidMarker, Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.specifiers import Specifier
from pip._vendor.packaging.utils import parse_sdist_filename, parse_wheel_filename
from pip._internal.exceptions import InstallationError
from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.req.req_file import ParsedRequirement
@@ -30,7 +30,14 @@ from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.filetypes import is_archive_file
from pip._internal.utils.misc import is_installable_dir
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.urls import path_to_url
from pip._internal.utils.pylock import (
package_archive_requirement_url,
package_directory_requirement_url,
package_sdist_requirement_url,
package_vcs_requirement_url,
package_wheel_requirement_url,
)
from pip._internal.utils.urls import path_to_url, strip_extras
from pip._internal.vcs import is_url, vcs
__all__ = [
@@ -40,25 +47,10 @@ __all__ = [
]
logger = logging.getLogger(__name__)
operators = Specifier._operators.keys()
def _strip_extras(path: str) -> tuple[str, str | None]:
m = re.match(r"^(.+)(\[[^\]]+\])$", path)
extras = None
if m:
path_no_extras = m.group(1).rstrip()
extras = m.group(2)
else:
path_no_extras = path
return path_no_extras, extras
def convert_extras(extras: str | None) -> set[str]:
if not extras:
return set()
return get_requirement("placeholder" + extras.lower()).extras
# All standard version specifier operators
# https://packaging.python.org/en/latest/specifications/version-specifiers/#id5
operators = ("~=", "==", "!=", "<=", ">=", "<", ">", "===")
def _set_requirement_extras(req: Requirement, new_extras: set[str]) -> Requirement:
@@ -108,7 +100,7 @@ def _parse_pip_syntax_editable(editable_req: str) -> tuple[str | None, str, set[
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
url_no_extras, extras = strip_extras(url)
if os.path.isdir(url_no_extras):
# Treating it as code that has already been checked out
@@ -116,14 +108,7 @@ def _parse_pip_syntax_editable(editable_req: str) -> tuple[str | None, str, set[
if url_no_extras.lower().startswith("file:"):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
get_requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, set()
return (package_name, url_no_extras, extras)
for version_control in vcs:
if url.lower().startswith(f"{version_control}:"):
@@ -255,7 +240,6 @@ def install_req_from_editable(
hash_options: dict[str, list[str]] | None = None,
constraint: bool = False,
user_supplied: bool = False,
permit_editable_wheels: bool = False,
config_settings: dict[str, str | list[str]] | None = None,
) -> InstallRequirement:
if constraint:
@@ -267,7 +251,6 @@ def install_req_from_editable(
comes_from=comes_from,
user_supplied=user_supplied,
editable=True,
permit_editable_wheels=permit_editable_wheels,
link=parts.link,
constraint=constraint,
isolated=isolated,
@@ -341,19 +324,22 @@ def parse_req_from_line(name: str, line_source: str | None) -> RequirementParts:
if not markers_as_string:
markers = None
else:
markers = Marker(markers_as_string)
try:
markers = Marker(markers_as_string)
except InvalidMarker as exc:
raise InstallationError(f"Invalid requirement: {name.strip()!r}: {exc}")
else:
markers = None
name = name.strip()
req_as_string = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras_as_string = None
if is_url(name):
link = Link(name)
extras: set[str] = set()
else:
p, extras_as_string = _strip_extras(path)
p, extras = strip_extras(path)
url = _get_url_from_path(p, name)
if url is not None:
link = Link(url)
@@ -376,8 +362,6 @@ def parse_req_from_line(name: str, line_source: str | None) -> RequirementParts:
else:
req_as_string = name
extras = convert_extras(extras_as_string)
def with_source(text: str) -> str:
if not line_source:
return text
@@ -453,23 +437,6 @@ def install_req_from_req_string(
except InvalidRequirement as exc:
raise InstallationError(f"Invalid requirement: {req_string!r}: {exc}")
domains_not_allowed = [
PyPI.file_storage_domain,
TestPyPI.file_storage_domain,
]
if (
req.url
and comes_from
and comes_from.link
and comes_from.link.netloc in domains_not_allowed
):
# Explicitly disallow pypi packages that depend on external urls
raise InstallationError(
"Packages installed from PyPI cannot depend on packages "
"which are not also hosted on PyPI.\n"
f"{comes_from.name} depends on {req} "
)
return InstallRequirement(
req,
comes_from,
@@ -546,7 +513,7 @@ def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement:
extras=[],
config_settings=ireq.config_settings,
user_supplied=ireq.user_supplied,
permit_editable_wheels=ireq.permit_editable_wheels,
source_link=ireq.source_link,
)
@@ -566,3 +533,95 @@ def install_req_extend_extras(
else None
)
return result
def _pylock_hashes_to_hash_options(hashes: Mapping[str, str]) -> dict[str, list[str]]:
return {k: [v] for k, v in hashes.items()}
def install_req_from_pylock_package(
package: pylock.Package,
package_dist: (
pylock.PackageVcs
| pylock.PackageArchive
| pylock.PackageDirectory
| pylock.PackageSdist
| pylock.PackageWheel
),
pylock_path_or_url: str,
user_supplied: bool,
) -> tuple[InstallRequirement, Link | None]:
"""Construct an InstallRequirement from a pylock package and artifact.
If the artifact is a Sdist or Wheel, also return a locked Link which
is meant to override candidates from indexes or --find-links.
"""
# TODO: validate file size
if isinstance(package_dist, pylock.PackageVcs):
req_url = package_vcs_requirement_url(pylock_path_or_url, package_dist)
return (
InstallRequirement(
req=Requirement(f"{package.name} @ {req_url}"),
comes_from=pylock_path_or_url,
user_supplied=user_supplied,
),
None,
)
elif isinstance(package_dist, pylock.PackageArchive):
req_url = package_archive_requirement_url(pylock_path_or_url, package_dist)
return (
InstallRequirement(
req=Requirement(f"{package.name} @ {req_url}"),
comes_from=pylock_path_or_url,
hash_options=_pylock_hashes_to_hash_options(package_dist.hashes),
user_supplied=user_supplied,
),
None,
)
elif isinstance(package_dist, pylock.PackageDirectory):
req_url = package_directory_requirement_url(pylock_path_or_url, package_dist)
if package_dist.editable:
return (
install_req_from_editable(
req_url,
comes_from=pylock_path_or_url,
user_supplied=user_supplied,
),
None,
)
else:
return (
install_req_from_line(
req_url,
comes_from=pylock_path_or_url,
user_supplied=user_supplied,
),
None,
)
else:
# wheel or sdist
version = package.version
if isinstance(package_dist, pylock.PackageWheel):
if not version:
_, version, _, _ = parse_wheel_filename(package_dist.filename)
requirement_url = package_wheel_requirement_url(
pylock_path_or_url, package_dist
)
elif isinstance(package_dist, pylock.PackageSdist):
if not version:
_, version = parse_sdist_filename(package_dist.filename)
requirement_url = package_sdist_requirement_url(
pylock_path_or_url, package_dist
)
ireq = InstallRequirement(
req=Requirement(f"{package.name}=={version}"),
comes_from=pylock_path_or_url,
hash_options=_pylock_hashes_to_hash_options(package_dist.hashes),
user_supplied=user_supplied,
)
locked_link = Link(
requirement_url,
comes_from=pylock_path_or_url,
upload_time=package_dist.upload_time,
)
return ireq, locked_link
@@ -14,7 +14,7 @@ class PEP723Exception(ValueError):
def pep723_metadata(scriptfile: str) -> dict[str, Any]:
with open(scriptfile) as f:
with open(scriptfile, encoding="utf8") as f:
script = f.read()
name = "script"
@@ -1,7 +1,8 @@
from collections.abc import Iterable, Iterator
from typing import Any
from pip._vendor.dependency_groups import DependencyGroupResolver
from pip._vendor.packaging.dependency_groups import DependencyGroupResolver
from pip._vendor.packaging.errors import ExceptionGroup
from pip._internal.exceptions import InstallationError
from pip._internal.utils.compat import tomllib
@@ -28,11 +29,13 @@ def _resolve_all_groups(
resolver = resolvers[path]
try:
yield from (str(req) for req in resolver.resolve(groupname))
except (ValueError, TypeError, LookupError) as e:
except ExceptionGroup as eg:
# Convert ExceptionGroup to a single InstallationError with all messages
messages = [str(e) for e in eg.exceptions]
raise InstallationError(
f"[dependency-groups] resolution failed for '{groupname}' "
f"from '{path}': {e}"
) from e
f"from '{path}': {'; '.join(messages)}"
) from eg
def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]:
@@ -54,7 +57,15 @@ def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]:
"Cannot resolve '--group' option."
)
resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
try:
resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
except ExceptionGroup as eg:
# Handle ExceptionGroup from resolver initialization
messages = [str(e) for e in eg.exceptions]
raise InstallationError(
f"[dependency-groups] data was invalid in {path}: {'; '.join(messages)}"
) from eg
return resolvers
@@ -5,7 +5,6 @@ Requirements file parsing
from __future__ import annotations
import codecs
import locale
import logging
import optparse
import os
@@ -13,13 +12,12 @@ import re
import shlex
import sys
import urllib.parse
from collections.abc import Generator, Iterable
from collections.abc import Callable, Generator, Iterable
from dataclasses import dataclass
from optparse import Values
from typing import (
TYPE_CHECKING,
Any,
Callable,
NoReturn,
)
@@ -27,6 +25,7 @@ from pip._internal.cli import cmdoptions
from pip._internal.exceptions import InstallationError, RequirementsFileParseError
from pip._internal.models.release_control import ReleaseControl
from pip._internal.models.search_scope import SearchScope
from pip._internal.utils.compat import get_locale_encoding
if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
@@ -59,6 +58,7 @@ SUPPORTED_OPTIONS: list[Callable[..., optparse.Option]] = [
cmdoptions.only_binary,
cmdoptions.prefer_binary,
cmdoptions.require_hashes,
cmdoptions.no_require_hashes,
cmdoptions.pre,
cmdoptions.all_releases,
cmdoptions.only_final,
@@ -101,18 +101,8 @@ DEFAULT_ENCODING = "utf-8"
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class ParsedRequirement:
# TODO: replace this with slots=True when dropping Python 3.9 support.
__slots__ = (
"requirement",
"is_editable",
"comes_from",
"constraint",
"options",
"line_source",
)
requirement: str
is_editable: bool
comes_from: str
@@ -121,10 +111,8 @@ class ParsedRequirement:
line_source: str | None
@dataclass(frozen=True)
@dataclass(frozen=True, slots=True)
class ParsedLine:
__slots__ = ("filename", "lineno", "args", "opts", "constraint")
filename: str
lineno: int
args: str
@@ -237,6 +225,8 @@ def handle_option_line(
# percolate options upward
if opts.require_hashes:
options.require_hashes = opts.require_hashes
if opts.no_require_hashes:
options.no_require_hashes = opts.no_require_hashes
if opts.features_enabled:
options.features_enabled.extend(
f for f in opts.features_enabled if f not in options.features_enabled
@@ -412,7 +402,7 @@ class RequirementsFileParser:
def _parse_file(
self, filename: str, constraint: bool
) -> Generator[ParsedLine, None, None]:
_, content = get_file_content(filename, self._session)
_, content = get_file_content(filename, self._session, constraint=constraint)
lines_enum = preprocess(content)
@@ -531,8 +521,6 @@ def join_lines(lines_enum: ReqFileLines) -> ReqFileLines:
assert primary_line_number is not None
yield primary_line_number, "".join(new_line)
# TODO: handle space after '\'.
def ignore_comments(lines_enum: ReqFileLines) -> ReqFileLines:
"""
@@ -572,7 +560,9 @@ def expand_env_variables(lines_enum: ReqFileLines) -> ReqFileLines:
yield line_number, line
def get_file_content(url: str, session: PipSession) -> tuple[str, str]:
def get_file_content(
url: str, session: PipSession, *, constraint: bool = False
) -> tuple[str, str]:
"""Gets the content of a file; it may be a filename, file: URL, or
http: URL. Returns (location, content). Content is unicode.
Respects # -*- coding: declarations on the retrieved files.
@@ -595,7 +585,8 @@ def get_file_content(url: str, session: PipSession) -> tuple[str, str]:
with open(url, "rb") as f:
raw_content = f.read()
except OSError as exc:
raise InstallationError(f"Could not open requirements file: {exc}")
kind = "constraint" if constraint else "requirements"
raise InstallationError(f"Could not open {kind} file: {exc}")
content = _decode_req_file(raw_content, url)
@@ -617,7 +608,7 @@ def _decode_req_file(data: bytes, url: str) -> str:
try:
return data.decode(DEFAULT_ENCODING)
except UnicodeDecodeError:
locale_encoding = locale.getpreferredencoding(False) or sys.getdefaultencoding()
locale_encoding = get_locale_encoding() or sys.getdefaultencoding()
logging.warning(
"unable to decode data from %s with default encoding %s, "
"falling back to encoding from locale: %s. "
@@ -34,7 +34,6 @@ from pip._internal.models.direct_url import DirectUrl
from pip._internal.models.link import Link
from pip._internal.operations.build.metadata import generate_metadata
from pip._internal.operations.build.metadata_editable import generate_editable_metadata
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
from pip._internal.req.req_uninstall import UninstallPathSet
from pip._internal.utils.deprecation import deprecated
@@ -49,9 +48,9 @@ from pip._internal.utils.misc import (
redact_auth_from_requirement,
redact_auth_from_url,
)
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.packaging import get_requirement, requirement_without_marker
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryKind
from pip._internal.utils.unpacking import unpack_file
from pip._internal.utils.virtualenv import running_under_virtualenv
from pip._internal.vcs import vcs
@@ -75,19 +74,18 @@ class InstallRequirement:
markers: Marker | None = None,
isolated: bool = False,
*,
source_link: Link | None = None,
hash_options: dict[str, list[str]] | None = None,
config_settings: dict[str, str | list[str]] | None = None,
constraint: bool = False,
extras: Collection[str] = (),
user_supplied: bool = False,
permit_editable_wheels: bool = False,
) -> None:
assert req is None or isinstance(req, Requirement), req
self.req = req
self.comes_from = comes_from
self.constraint = constraint
self.editable = editable
self.permit_editable_wheels = permit_editable_wheels
# source_dir is the local directory where the linked requirement is
# located, or unpacked. In case unpacking is needed, creating and
@@ -107,6 +105,10 @@ class InstallRequirement:
link = Link(req.url)
self.link = self.original_link = link
# The link selected before wheel-cache substitution. Unlike ``link``, this
# does not change when pip substitutes a locally built cached wheel.
self._source_link = source_link if source_link is not None else link
# When this InstallRequirement is a wheel obtained from the cache of locally
# built wheels, this is the source link corresponding to the cache entry, which
# was used to download and build the cached wheel.
@@ -170,6 +172,8 @@ class InstallRequirement:
self.requirements_to_check: list[str] = []
# The PEP 517 backend we should use to build the project
self._pep517_backend_spec: str
self._pep517_backend_path: str | None
self.pep517_backend: BuildBackendHookCaller | None = None
# This requirement needs more preparation before it can be built
@@ -179,8 +183,20 @@ class InstallRequirement:
self._archive_source: Path | None = None
def __str__(self) -> str:
return self._format()
def format_without_marker(self) -> str:
"""Like ``str(self)``, but without the requirement's environment marker.
For error messages, where markers such as ``extra == "foo"`` are noise.
``self.req`` and ``self.markers`` are not modified.
"""
return self._format(include_marker=False)
def _format(self, include_marker: bool = True) -> str:
if self.req:
s = redact_auth_from_requirement(self.req)
req = self.req if include_marker else requirement_without_marker(self.req)
s = redact_auth_from_requirement(req)
if self.link:
s += f" from {redact_auth_from_url(self.link.url)}"
elif self.link:
@@ -229,12 +245,13 @@ class InstallRequirement:
@functools.cached_property
def supports_pyproject_editable(self) -> bool:
assert self.pep517_backend
assert isinstance(self.pep517_backend, ConfiguredBuildBackendHookCaller)
with self.build_env:
runner = runner_with_spinner_message(
"Checking if build backend supports build_editable"
)
with self.pep517_backend.subprocess_runner(runner):
return "build_editable" in self.pep517_backend._supported_features()
return self.pep517_backend.supports_feature("build_editable")
@property
def specifier(self) -> SpecifierSet:
@@ -246,6 +263,19 @@ class InstallRequirement:
"""Whether this requirement was specified as a direct URL."""
return self.original_link is not None
@property
def source_link(self) -> Link | None:
"""The link selected before wheel-cache substitution."""
return self._source_link
def set_source_link(self, link: Link) -> None:
"""Record the selected source link, rejecting later changes."""
if self._source_link is None:
self._source_link = link
return
if self._source_link != link:
raise InstallationError(f"Cannot change the source link for {self}.")
@property
def is_pinned(self) -> bool:
"""Return whether I am pinned to an exact version.
@@ -332,7 +362,7 @@ class InstallRequirement:
# builds (such as numpy). Thus, we ensure that the real path
# is returned.
self._temp_build_dir = TempDirectory(
kind=tempdir_kinds.REQ_BUILD, globally_managed=True
kind=TempDirectoryKind.REQ_BUILD, globally_managed=True
)
return self._temp_build_dir.path
@@ -359,7 +389,7 @@ class InstallRequirement:
return TempDirectory(
path=actual_build_dir,
delete=delete_arg,
kind=tempdir_kinds.REQ_BUILD,
kind=TempDirectoryKind.REQ_BUILD,
globally_managed=True,
).path
@@ -486,11 +516,26 @@ class InstallRequirement:
requires, backend, check, backend_path = pyproject_toml_data
self.requirements_to_check = check
self.pyproject_requires = requires
self._pep517_backend_spec = backend
self._pep517_backend_path = backend_path
def configure_backend(self, python_executable: str) -> None:
"""Set up the build backend hook caller.
This is done separately after pyproject.toml loading as the backend
need to be called with the build environment's Python executable,
which can vary."""
self.pep517_backend = ConfiguredBuildBackendHookCaller(
self,
self.unpacked_source_directory,
backend,
backend_path=backend_path,
self._pep517_backend_spec,
backend_path=self._pep517_backend_path,
python_executable=python_executable,
backend_warning_level=(
logging.WARNING
if self.link and self.link.is_existing_dir()
else logging.DEBUG
),
)
def editable_sanity_check(self) -> None:
@@ -506,7 +551,7 @@ class InstallRequirement:
f"Consider using a build backend that supports PEP 660."
)
def prepare_metadata(self) -> None:
def prepare_metadata(self, allow_editables: bool) -> None:
"""Ensure that project metadata is available.
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
@@ -516,11 +561,7 @@ class InstallRequirement:
details = self.name or f"from {self.link}"
assert self.pep517_backend is not None
if (
self.editable
and self.permit_editable_wheels
and self.supports_pyproject_editable
):
if self.editable and allow_editables and self.supports_pyproject_editable:
self.metadata_directory = generate_editable_metadata(
build_env=self.build_env,
backend=self.pep517_backend,
@@ -764,7 +805,17 @@ class InstallRequirement:
warn_script_location: bool = True,
use_user_site: bool = False,
pycompile: bool = True,
script_executable: str | None = None,
*,
rollback_on_error: bool = False,
) -> None:
# Lazy import to avoid transitively importing `_vendor.distlib.compat`
# which in turn imports `urllib.request` which is slow.
# During an actual installation, `urllib.request` will end up imported anyway,
# but `req.req_install` (this module) is also imported from commands that
# don't actually install anything (e.g. `pip freeze` or `pip show`).
from pip._internal.operations.install.wheel import install_wheel
assert self.req is not None
scheme = get_scheme(
self.req.name,
@@ -787,6 +838,8 @@ class InstallRequirement:
warn_script_location=warn_script_location,
direct_url=self.download_info if self.is_direct else None,
requested=self.user_supplied,
script_executable=script_executable,
rollback_on_error=rollback_on_error,
)
self.install_succeeded = True
@@ -2,19 +2,24 @@ from __future__ import annotations
import functools
import os
import shutil
import sys
import sysconfig
from collections.abc import Generator, Iterable
from collections.abc import Callable, Generator, Iterable
from importlib.util import cache_from_source
from typing import Any, Callable
from typing import Any
from pip._internal.exceptions import LegacyDistutilsInstall, UninstallMissingRecord
from pip._internal.exceptions import (
BlockedImportError,
LegacyDistutilsInstall,
UninstallMissingRecord,
)
from pip._internal.locations import get_bin_prefix, get_bin_user
from pip._internal.metadata import BaseDistribution
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.logging import getLogger, indent_log
from pip._internal.utils.misc import ask, normalize_path, renames, rmtree
from pip._internal.utils.misc import ask, normalize_path, rmtree
from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
from pip._internal.utils.virtualenv import running_under_virtualenv
@@ -96,14 +101,19 @@ def compact(paths: Iterable[str]) -> set[str]:
sep = os.path.sep
short_paths: set[str] = set()
prefixes: set[str] = set()
for path in sorted(paths, key=len):
should_skip = any(
path.startswith(shortpath.rstrip("*"))
and path[len(shortpath.rstrip("*").rstrip(sep))] == sep
for shortpath in short_paths
)
if not should_skip:
current = path[:-1] if path.endswith(sep + "*") else path
parent = current.rstrip(sep)
while parent:
if parent in prefixes:
break
next_parent = os.path.dirname(parent).rstrip(sep)
parent = "" if next_parent == parent else next_parent
else:
short_paths.add(path)
prefixes.add(current.rstrip(sep))
return short_paths
@@ -201,6 +211,8 @@ class StashedUninstallPathSet:
# (old path, new path) tuples for each move that may need
# to be undone.
self._moves: list[tuple[str, str]] = []
# Interrupted transfers must not be retried automatically.
self._manual_recovery: list[tuple[str, str]] = []
def _get_directory_stash(self, path: str) -> str:
"""Stashes a directory.
@@ -244,52 +256,103 @@ class StashedUninstallPathSet:
return save_dir.path
def stash(self, path: str) -> str:
"""Stashes the directory or file and returns its new location.
Handle symlinks as files to avoid modifying the symlink targets.
"""
"""Move a path into temporary storage and return its new location."""
path_is_dir = os.path.isdir(path) and not os.path.islink(path)
if path_is_dir:
new_path = self._get_directory_stash(path)
else:
new_path = self._get_file_stash(path)
self._moves.append((path, new_path))
if path_is_dir and os.path.isdir(new_path):
# If we're moving a directory, we need to
# remove the destination first or else it will be
# moved to inside the existing directory.
# We just created new_path ourselves, so it will
# be removable.
# Remove the empty placeholder so the directory moves to this exact path.
os.rmdir(new_path)
renames(path, new_path)
parent = os.path.dirname(new_path)
if not os.path.exists(parent):
os.makedirs(parent)
try:
shutil.move(path, new_path)
except BaseException:
if os.path.lexists(new_path):
# A failed move may leave either path with only part of the data.
self._manual_recovery.append((path, new_path))
raise
self._moves.append((path, new_path))
old_parent, name = os.path.split(path)
if old_parent and name:
try:
os.removedirs(old_parent)
except OSError:
pass
return new_path
def commit(self) -> None:
"""Commits the uninstall by removing stashed files."""
"""Remove backups unless an interrupted transfer needs manual recovery."""
if self._manual_recovery:
return
for save_dir in self._save_dirs.values():
save_dir.cleanup()
self._moves = []
self._save_dirs = {}
def _restore(self, original: str, stashed: str) -> None:
"""Restore a completed stash and record failures for manual recovery."""
try:
# Check the backup before removing a replacement.
os.lstat(stashed)
if os.path.isfile(original) or os.path.islink(original):
os.unlink(original)
elif os.path.isdir(original):
rmtree(original)
parent, name = os.path.split(original)
if parent and name and not os.path.exists(parent):
os.makedirs(parent)
shutil.move(stashed, original)
except BaseException:
self._manual_recovery.append((original, stashed))
raise
def rollback(self) -> None:
"""Undoes the uninstall by moving stashed files back."""
"""Restore stashed files, keeping backups when restoration fails."""
for p in self._moves:
logger.info("Moving to %s\n from %s", *p)
for new_path, path in self._moves:
try:
logger.debug("Replacing %s from %s", new_path, path)
if os.path.isfile(new_path) or os.path.islink(new_path):
os.unlink(new_path)
elif os.path.isdir(new_path):
rmtree(new_path)
renames(path, new_path)
except OSError as ex:
logger.error("Failed to restore %s", new_path)
logger.debug("Exception: %s", ex)
attempted: set[tuple[str, str]] = set()
try:
for original, stashed in self._moves:
try:
logger.debug("Replacing %s from %s", original, stashed)
attempted.add((original, stashed))
self._restore(original, stashed)
except OSError as ex:
logger.debug("Exception: %s", ex)
finally:
# Retrying an interrupted move could overwrite the complete copy.
self._moves = [move for move in self._moves if move not in attempted]
self.commit()
def report_failed_recovery(self) -> None:
"""Report retained backups that still need manual restoration."""
for original, backup in self.recovery_paths:
logger.error(
"Could not restore %r from %r. Check both paths before retrying.",
original,
backup,
)
@property
def recovery_paths(self) -> tuple[tuple[str, str], ...]:
"""Return original paths and their pending backup locations."""
return tuple(self._moves + self._manual_recovery)
@property
def can_rollback(self) -> bool:
return bool(self._moves)
@@ -369,14 +432,22 @@ class UninstallPathSet:
for_rename = compress_for_rename(self._paths)
for path in sorted(compact(for_rename)):
moved.stash(path)
logger.verbose("Removing file or directory %s", path)
try:
for path in sorted(compact(for_rename)):
moved.stash(path)
logger.verbose("Removing file or directory %s", path)
for pth in self._pth.values():
pth.remove()
for pth in self._pth.values():
pth.remove()
logger.info("Successfully uninstalled %s", dist_name_version)
logger.info("Successfully uninstalled %s", dist_name_version)
except (OSError, BlockedImportError):
if moved.can_rollback:
self.rollback()
else:
moved.commit()
moved.report_failed_recovery()
raise
def _allowed_to_proceed(self, verbose: bool) -> bool:
"""Display which files would be deleted and prompt for confirmation"""
@@ -406,16 +477,24 @@ class UninstallPathSet:
return ask("Proceed (Y/n)? ", ("y", "n", "")) != "n"
@property
def can_rollback(self) -> bool:
return self._moved_paths.can_rollback
def rollback(self) -> None:
"""Rollback the changes previously made by remove()."""
if not self._moved_paths.can_rollback:
# Manual file recovery can still leave saved .pth entries to restore.
if not self.can_rollback and not self._moved_paths.recovery_paths:
logger.error(
"Can't roll back %s; was not uninstalled",
self._dist.raw_name,
)
return
logger.info("Rolling back uninstall of %s", self._dist.raw_name)
self._moved_paths.rollback()
try:
self._moved_paths.rollback()
finally:
self._moved_paths.report_failed_recovery()
for pth in self._pth.values():
pth.rollback()
@@ -609,23 +688,26 @@ class UninstallPthEntries:
if not os.path.isfile(self.file):
logger.warning("Cannot remove entries from nonexistent file %s", self.file)
return
with open(self.file, "rb") as fh:
# windows uses '\r\n' with py3k, but uses '\n' with py2.x
lines = fh.readlines()
self._saved_lines = lines
self._saved_lines = lines.copy()
if any(b"\r\n" in line for line in lines):
endline = "\r\n"
else:
endline = "\n"
# handle missing trailing newline
# Match entries even when the last line has no newline.
if lines and not lines[-1].endswith(endline.encode("utf-8")):
lines[-1] = lines[-1] + endline.encode("utf-8")
for entry in self.entries:
try:
logger.verbose("Removing entry: %s", entry)
lines.remove((entry + endline).encode("utf-8"))
except ValueError:
pass
with open(self.file, "wb") as fh:
fh.writelines(lines)
@@ -633,7 +715,15 @@ class UninstallPthEntries:
if self._saved_lines is None:
logger.error("Cannot roll back changes to %s, none were made", self.file)
return False
logger.debug("Rolling %s back to previous state", self.file)
with open(self.file, "wb") as fh:
fh.writelines(self._saved_lines)
try:
with open(self.file, "wb") as fh:
fh.writelines(self._saved_lines)
except (OSError, BlockedImportError):
logger.error(
"Could not restore %r. Check this file before retrying.", self.file
)
raise
return True