You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

req_install.py 39KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021
  1. from __future__ import absolute_import
  2. import logging
  3. import os
  4. import shutil
  5. import sys
  6. import sysconfig
  7. import zipfile
  8. from distutils.util import change_root
  9. from pip._vendor import pkg_resources, six
  10. from pip._vendor.packaging.requirements import Requirement
  11. from pip._vendor.packaging.utils import canonicalize_name
  12. from pip._vendor.packaging.version import Version
  13. from pip._vendor.packaging.version import parse as parse_version
  14. from pip._vendor.pep517.wrappers import Pep517HookCaller
  15. from pip._internal import wheel
  16. from pip._internal.build_env import NoOpBuildEnvironment
  17. from pip._internal.exceptions import InstallationError
  18. from pip._internal.locations import (
  19. PIP_DELETE_MARKER_FILENAME, running_under_virtualenv,
  20. )
  21. from pip._internal.models.link import Link
  22. from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
  23. from pip._internal.req.req_uninstall import UninstallPathSet
  24. from pip._internal.utils.compat import native_str
  25. from pip._internal.utils.hashes import Hashes
  26. from pip._internal.utils.logging import indent_log
  27. from pip._internal.utils.misc import (
  28. _make_build_dir, ask_path_exists, backup_dir, call_subprocess,
  29. display_path, dist_in_site_packages, dist_in_usersite, ensure_dir,
  30. get_installed_version, redact_password_from_url, rmtree,
  31. )
  32. from pip._internal.utils.packaging import get_metadata
  33. from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
  34. from pip._internal.utils.temp_dir import TempDirectory
  35. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  36. from pip._internal.utils.ui import open_spinner
  37. from pip._internal.vcs import vcs
  38. from pip._internal.wheel import move_wheel_files
  39. if MYPY_CHECK_RUNNING:
  40. from typing import ( # noqa: F401
  41. Optional, Iterable, List, Union, Any, Text, Sequence, Dict
  42. )
  43. from pip._internal.build_env import BuildEnvironment # noqa: F401
  44. from pip._internal.cache import WheelCache # noqa: F401
  45. from pip._internal.index import PackageFinder # noqa: F401
  46. from pip._vendor.pkg_resources import Distribution # noqa: F401
  47. from pip._vendor.packaging.specifiers import SpecifierSet # noqa: F401
  48. from pip._vendor.packaging.markers import Marker # noqa: F401
  49. logger = logging.getLogger(__name__)
  50. class InstallRequirement(object):
  51. """
  52. Represents something that may be installed later on, may have information
  53. about where to fetch the relavant requirement and also contains logic for
  54. installing the said requirement.
  55. """
  56. def __init__(
  57. self,
  58. req, # type: Optional[Requirement]
  59. comes_from, # type: Optional[Union[str, InstallRequirement]]
  60. source_dir=None, # type: Optional[str]
  61. editable=False, # type: bool
  62. link=None, # type: Optional[Link]
  63. update=True, # type: bool
  64. markers=None, # type: Optional[Marker]
  65. use_pep517=None, # type: Optional[bool]
  66. isolated=False, # type: bool
  67. options=None, # type: Optional[Dict[str, Any]]
  68. wheel_cache=None, # type: Optional[WheelCache]
  69. constraint=False, # type: bool
  70. extras=() # type: Iterable[str]
  71. ):
  72. # type: (...) -> None
  73. assert req is None or isinstance(req, Requirement), req
  74. self.req = req
  75. self.comes_from = comes_from
  76. self.constraint = constraint
  77. if source_dir is not None:
  78. self.source_dir = os.path.normpath(os.path.abspath(source_dir))
  79. else:
  80. self.source_dir = None
  81. self.editable = editable
  82. self._wheel_cache = wheel_cache
  83. if link is None and req and req.url:
  84. # PEP 508 URL requirement
  85. link = Link(req.url)
  86. self.link = self.original_link = link
  87. if extras:
  88. self.extras = extras
  89. elif req:
  90. self.extras = {
  91. pkg_resources.safe_extra(extra) for extra in req.extras
  92. }
  93. else:
  94. self.extras = set()
  95. if markers is None and req:
  96. markers = req.marker
  97. self.markers = markers
  98. self._egg_info_path = None # type: Optional[str]
  99. # This holds the pkg_resources.Distribution object if this requirement
  100. # is already available:
  101. self.satisfied_by = None
  102. # This hold the pkg_resources.Distribution object if this requirement
  103. # conflicts with another installed distribution:
  104. self.conflicts_with = None
  105. # Temporary build location
  106. self._temp_build_dir = TempDirectory(kind="req-build")
  107. # Used to store the global directory where the _temp_build_dir should
  108. # have been created. Cf _correct_build_location method.
  109. self._ideal_build_dir = None # type: Optional[str]
  110. # True if the editable should be updated:
  111. self.update = update
  112. # Set to True after successful installation
  113. self.install_succeeded = None # type: Optional[bool]
  114. # UninstallPathSet of uninstalled distribution (for possible rollback)
  115. self.uninstalled_pathset = None
  116. self.options = options if options else {}
  117. # Set to True after successful preparation of this requirement
  118. self.prepared = False
  119. self.is_direct = False
  120. self.isolated = isolated
  121. self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment
  122. # For PEP 517, the directory where we request the project metadata
  123. # gets stored. We need this to pass to build_wheel, so the backend
  124. # can ensure that the wheel matches the metadata (see the PEP for
  125. # details).
  126. self.metadata_directory = None # type: Optional[str]
  127. # The static build requirements (from pyproject.toml)
  128. self.pyproject_requires = None # type: Optional[List[str]]
  129. # Build requirements that we will check are available
  130. self.requirements_to_check = [] # type: List[str]
  131. # The PEP 517 backend we should use to build the project
  132. self.pep517_backend = None # type: Optional[Pep517HookCaller]
  133. # Are we using PEP 517 for this requirement?
  134. # After pyproject.toml has been loaded, the only valid values are True
  135. # and False. Before loading, None is valid (meaning "use the default").
  136. # Setting an explicit value before loading pyproject.toml is supported,
  137. # but after loading this flag should be treated as read only.
  138. self.use_pep517 = use_pep517
  139. def __str__(self):
  140. if self.req:
  141. s = str(self.req)
  142. if self.link:
  143. s += ' from %s' % redact_password_from_url(self.link.url)
  144. elif self.link:
  145. s = redact_password_from_url(self.link.url)
  146. else:
  147. s = '<InstallRequirement>'
  148. if self.satisfied_by is not None:
  149. s += ' in %s' % display_path(self.satisfied_by.location)
  150. if self.comes_from:
  151. if isinstance(self.comes_from, six.string_types):
  152. comes_from = self.comes_from
  153. else:
  154. comes_from = self.comes_from.from_path()
  155. if comes_from:
  156. s += ' (from %s)' % comes_from
  157. return s
  158. def __repr__(self):
  159. return '<%s object: %s editable=%r>' % (
  160. self.__class__.__name__, str(self), self.editable)
  161. def populate_link(self, finder, upgrade, require_hashes):
  162. # type: (PackageFinder, bool, bool) -> None
  163. """Ensure that if a link can be found for this, that it is found.
  164. Note that self.link may still be None - if Upgrade is False and the
  165. requirement is already installed.
  166. If require_hashes is True, don't use the wheel cache, because cached
  167. wheels, always built locally, have different hashes than the files
  168. downloaded from the index server and thus throw false hash mismatches.
  169. Furthermore, cached wheels at present have undeterministic contents due
  170. to file modification times.
  171. """
  172. if self.link is None:
  173. self.link = finder.find_requirement(self, upgrade)
  174. if self._wheel_cache is not None and not require_hashes:
  175. old_link = self.link
  176. self.link = self._wheel_cache.get(self.link, self.name)
  177. if old_link != self.link:
  178. logger.debug('Using cached wheel link: %s', self.link)
  179. # Things that are valid for all kinds of requirements?
  180. @property
  181. def name(self):
  182. # type: () -> Optional[str]
  183. if self.req is None:
  184. return None
  185. return native_str(pkg_resources.safe_name(self.req.name))
  186. @property
  187. def specifier(self):
  188. # type: () -> SpecifierSet
  189. return self.req.specifier
  190. @property
  191. def is_pinned(self):
  192. # type: () -> bool
  193. """Return whether I am pinned to an exact version.
  194. For example, some-package==1.2 is pinned; some-package>1.2 is not.
  195. """
  196. specifiers = self.specifier
  197. return (len(specifiers) == 1 and
  198. next(iter(specifiers)).operator in {'==', '==='})
  199. @property
  200. def installed_version(self):
  201. return get_installed_version(self.name)
  202. def match_markers(self, extras_requested=None):
  203. # type: (Optional[Iterable[str]]) -> bool
  204. if not extras_requested:
  205. # Provide an extra to safely evaluate the markers
  206. # without matching any extra
  207. extras_requested = ('',)
  208. if self.markers is not None:
  209. return any(
  210. self.markers.evaluate({'extra': extra})
  211. for extra in extras_requested)
  212. else:
  213. return True
  214. @property
  215. def has_hash_options(self):
  216. # type: () -> bool
  217. """Return whether any known-good hashes are specified as options.
  218. These activate --require-hashes mode; hashes specified as part of a
  219. URL do not.
  220. """
  221. return bool(self.options.get('hashes', {}))
  222. def hashes(self, trust_internet=True):
  223. # type: (bool) -> Hashes
  224. """Return a hash-comparer that considers my option- and URL-based
  225. hashes to be known-good.
  226. Hashes in URLs--ones embedded in the requirements file, not ones
  227. downloaded from an index server--are almost peers with ones from
  228. flags. They satisfy --require-hashes (whether it was implicitly or
  229. explicitly activated) but do not activate it. md5 and sha224 are not
  230. allowed in flags, which should nudge people toward good algos. We
  231. always OR all hashes together, even ones from URLs.
  232. :param trust_internet: Whether to trust URL-based (#md5=...) hashes
  233. downloaded from the internet, as by populate_link()
  234. """
  235. good_hashes = self.options.get('hashes', {}).copy()
  236. link = self.link if trust_internet else self.original_link
  237. if link and link.hash:
  238. good_hashes.setdefault(link.hash_name, []).append(link.hash)
  239. return Hashes(good_hashes)
  240. def from_path(self):
  241. # type: () -> Optional[str]
  242. """Format a nice indicator to show where this "comes from"
  243. """
  244. if self.req is None:
  245. return None
  246. s = str(self.req)
  247. if self.comes_from:
  248. if isinstance(self.comes_from, six.string_types):
  249. comes_from = self.comes_from
  250. else:
  251. comes_from = self.comes_from.from_path()
  252. if comes_from:
  253. s += '->' + comes_from
  254. return s
  255. def build_location(self, build_dir):
  256. # type: (str) -> Optional[str]
  257. assert build_dir is not None
  258. if self._temp_build_dir.path is not None:
  259. return self._temp_build_dir.path
  260. if self.req is None:
  261. # for requirement via a path to a directory: the name of the
  262. # package is not available yet so we create a temp directory
  263. # Once run_egg_info will have run, we'll be able
  264. # to fix it via _correct_build_location
  265. # Some systems have /tmp as a symlink which confuses custom
  266. # builds (such as numpy). Thus, we ensure that the real path
  267. # is returned.
  268. self._temp_build_dir.create()
  269. self._ideal_build_dir = build_dir
  270. return self._temp_build_dir.path
  271. if self.editable:
  272. name = self.name.lower()
  273. else:
  274. name = self.name
  275. # FIXME: Is there a better place to create the build_dir? (hg and bzr
  276. # need this)
  277. if not os.path.exists(build_dir):
  278. logger.debug('Creating directory %s', build_dir)
  279. _make_build_dir(build_dir)
  280. return os.path.join(build_dir, name)
  281. def _correct_build_location(self):
  282. # type: () -> None
  283. """Move self._temp_build_dir to self._ideal_build_dir/self.req.name
  284. For some requirements (e.g. a path to a directory), the name of the
  285. package is not available until we run egg_info, so the build_location
  286. will return a temporary directory and store the _ideal_build_dir.
  287. This is only called by self.run_egg_info to fix the temporary build
  288. directory.
  289. """
  290. if self.source_dir is not None:
  291. return
  292. assert self.req is not None
  293. assert self._temp_build_dir.path
  294. assert (self._ideal_build_dir is not None and
  295. self._ideal_build_dir.path) # type: ignore
  296. old_location = self._temp_build_dir.path
  297. self._temp_build_dir.path = None
  298. new_location = self.build_location(self._ideal_build_dir)
  299. if os.path.exists(new_location):
  300. raise InstallationError(
  301. 'A package already exists in %s; please remove it to continue'
  302. % display_path(new_location))
  303. logger.debug(
  304. 'Moving package %s from %s to new location %s',
  305. self, display_path(old_location), display_path(new_location),
  306. )
  307. shutil.move(old_location, new_location)
  308. self._temp_build_dir.path = new_location
  309. self._ideal_build_dir = None
  310. self.source_dir = os.path.normpath(os.path.abspath(new_location))
  311. self._egg_info_path = None
  312. # Correct the metadata directory, if it exists
  313. if self.metadata_directory:
  314. old_meta = self.metadata_directory
  315. rel = os.path.relpath(old_meta, start=old_location)
  316. new_meta = os.path.join(new_location, rel)
  317. new_meta = os.path.normpath(os.path.abspath(new_meta))
  318. self.metadata_directory = new_meta
  319. def remove_temporary_source(self):
  320. # type: () -> None
  321. """Remove the source files from this requirement, if they are marked
  322. for deletion"""
  323. if self.source_dir and os.path.exists(
  324. os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME)):
  325. logger.debug('Removing source in %s', self.source_dir)
  326. rmtree(self.source_dir)
  327. self.source_dir = None
  328. self._temp_build_dir.cleanup()
  329. self.build_env.cleanup()
  330. def check_if_exists(self, use_user_site):
  331. # type: (bool) -> bool
  332. """Find an installed distribution that satisfies or conflicts
  333. with this requirement, and set self.satisfied_by or
  334. self.conflicts_with appropriately.
  335. """
  336. if self.req is None:
  337. return False
  338. try:
  339. # get_distribution() will resolve the entire list of requirements
  340. # anyway, and we've already determined that we need the requirement
  341. # in question, so strip the marker so that we don't try to
  342. # evaluate it.
  343. no_marker = Requirement(str(self.req))
  344. no_marker.marker = None
  345. self.satisfied_by = pkg_resources.get_distribution(str(no_marker))
  346. if self.editable and self.satisfied_by:
  347. self.conflicts_with = self.satisfied_by
  348. # when installing editables, nothing pre-existing should ever
  349. # satisfy
  350. self.satisfied_by = None
  351. return True
  352. except pkg_resources.DistributionNotFound:
  353. return False
  354. except pkg_resources.VersionConflict:
  355. existing_dist = pkg_resources.get_distribution(
  356. self.req.name
  357. )
  358. if use_user_site:
  359. if dist_in_usersite(existing_dist):
  360. self.conflicts_with = existing_dist
  361. elif (running_under_virtualenv() and
  362. dist_in_site_packages(existing_dist)):
  363. raise InstallationError(
  364. "Will not install to the user site because it will "
  365. "lack sys.path precedence to %s in %s" %
  366. (existing_dist.project_name, existing_dist.location)
  367. )
  368. else:
  369. self.conflicts_with = existing_dist
  370. return True
  371. # Things valid for wheels
  372. @property
  373. def is_wheel(self):
  374. # type: () -> bool
  375. if not self.link:
  376. return False
  377. return self.link.is_wheel
  378. def move_wheel_files(
  379. self,
  380. wheeldir, # type: str
  381. root=None, # type: Optional[str]
  382. home=None, # type: Optional[str]
  383. prefix=None, # type: Optional[str]
  384. warn_script_location=True, # type: bool
  385. use_user_site=False, # type: bool
  386. pycompile=True # type: bool
  387. ):
  388. # type: (...) -> None
  389. move_wheel_files(
  390. self.name, self.req, wheeldir,
  391. user=use_user_site,
  392. home=home,
  393. root=root,
  394. prefix=prefix,
  395. pycompile=pycompile,
  396. isolated=self.isolated,
  397. warn_script_location=warn_script_location,
  398. )
  399. # Things valid for sdists
  400. @property
  401. def setup_py_dir(self):
  402. # type: () -> str
  403. return os.path.join(
  404. self.source_dir,
  405. self.link and self.link.subdirectory_fragment or '')
  406. @property
  407. def setup_py(self):
  408. # type: () -> str
  409. assert self.source_dir, "No source dir for %s" % self
  410. setup_py = os.path.join(self.setup_py_dir, 'setup.py')
  411. # Python2 __file__ should not be unicode
  412. if six.PY2 and isinstance(setup_py, six.text_type):
  413. setup_py = setup_py.encode(sys.getfilesystemencoding())
  414. return setup_py
  415. @property
  416. def pyproject_toml(self):
  417. # type: () -> str
  418. assert self.source_dir, "No source dir for %s" % self
  419. return make_pyproject_path(self.setup_py_dir)
  420. def load_pyproject_toml(self):
  421. # type: () -> None
  422. """Load the pyproject.toml file.
  423. After calling this routine, all of the attributes related to PEP 517
  424. processing for this requirement have been set. In particular, the
  425. use_pep517 attribute can be used to determine whether we should
  426. follow the PEP 517 or legacy (setup.py) code path.
  427. """
  428. pep517_data = load_pyproject_toml(
  429. self.use_pep517,
  430. self.pyproject_toml,
  431. self.setup_py,
  432. str(self)
  433. )
  434. if pep517_data is None:
  435. self.use_pep517 = False
  436. else:
  437. self.use_pep517 = True
  438. requires, backend, check = pep517_data
  439. self.requirements_to_check = check
  440. self.pyproject_requires = requires
  441. self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend)
  442. # Use a custom function to call subprocesses
  443. self.spin_message = ""
  444. def runner(cmd, cwd=None, extra_environ=None):
  445. with open_spinner(self.spin_message) as spinner:
  446. call_subprocess(
  447. cmd,
  448. cwd=cwd,
  449. extra_environ=extra_environ,
  450. show_stdout=False,
  451. spinner=spinner
  452. )
  453. self.spin_message = ""
  454. self.pep517_backend._subprocess_runner = runner
  455. def prepare_metadata(self):
  456. # type: () -> None
  457. """Ensure that project metadata is available.
  458. Under PEP 517, call the backend hook to prepare the metadata.
  459. Under legacy processing, call setup.py egg-info.
  460. """
  461. assert self.source_dir
  462. with indent_log():
  463. if self.use_pep517:
  464. self.prepare_pep517_metadata()
  465. else:
  466. self.run_egg_info()
  467. if not self.req:
  468. if isinstance(parse_version(self.metadata["Version"]), Version):
  469. op = "=="
  470. else:
  471. op = "==="
  472. self.req = Requirement(
  473. "".join([
  474. self.metadata["Name"],
  475. op,
  476. self.metadata["Version"],
  477. ])
  478. )
  479. self._correct_build_location()
  480. else:
  481. metadata_name = canonicalize_name(self.metadata["Name"])
  482. if canonicalize_name(self.req.name) != metadata_name:
  483. logger.warning(
  484. 'Generating metadata for package %s '
  485. 'produced metadata for project name %s. Fix your '
  486. '#egg=%s fragments.',
  487. self.name, metadata_name, self.name
  488. )
  489. self.req = Requirement(metadata_name)
  490. def prepare_pep517_metadata(self):
  491. # type: () -> None
  492. assert self.pep517_backend is not None
  493. metadata_dir = os.path.join(
  494. self.setup_py_dir,
  495. 'pip-wheel-metadata'
  496. )
  497. ensure_dir(metadata_dir)
  498. with self.build_env:
  499. # Note that Pep517HookCaller implements a fallback for
  500. # prepare_metadata_for_build_wheel, so we don't have to
  501. # consider the possibility that this hook doesn't exist.
  502. backend = self.pep517_backend
  503. self.spin_message = "Preparing wheel metadata"
  504. distinfo_dir = backend.prepare_metadata_for_build_wheel(
  505. metadata_dir
  506. )
  507. self.metadata_directory = os.path.join(metadata_dir, distinfo_dir)
  508. def run_egg_info(self):
  509. # type: () -> None
  510. if self.name:
  511. logger.debug(
  512. 'Running setup.py (path:%s) egg_info for package %s',
  513. self.setup_py, self.name,
  514. )
  515. else:
  516. logger.debug(
  517. 'Running setup.py (path:%s) egg_info for package from %s',
  518. self.setup_py, self.link,
  519. )
  520. script = SETUPTOOLS_SHIM % self.setup_py
  521. base_cmd = [sys.executable, '-c', script]
  522. if self.isolated:
  523. base_cmd += ["--no-user-cfg"]
  524. egg_info_cmd = base_cmd + ['egg_info']
  525. # We can't put the .egg-info files at the root, because then the
  526. # source code will be mistaken for an installed egg, causing
  527. # problems
  528. if self.editable:
  529. egg_base_option = [] # type: List[str]
  530. else:
  531. egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info')
  532. ensure_dir(egg_info_dir)
  533. egg_base_option = ['--egg-base', 'pip-egg-info']
  534. with self.build_env:
  535. call_subprocess(
  536. egg_info_cmd + egg_base_option,
  537. cwd=self.setup_py_dir,
  538. show_stdout=False,
  539. command_desc='python setup.py egg_info')
  540. @property
  541. def egg_info_path(self):
  542. # type: () -> str
  543. if self._egg_info_path is None:
  544. if self.editable:
  545. base = self.source_dir
  546. else:
  547. base = os.path.join(self.setup_py_dir, 'pip-egg-info')
  548. filenames = os.listdir(base)
  549. if self.editable:
  550. filenames = []
  551. for root, dirs, files in os.walk(base):
  552. for dir in vcs.dirnames:
  553. if dir in dirs:
  554. dirs.remove(dir)
  555. # Iterate over a copy of ``dirs``, since mutating
  556. # a list while iterating over it can cause trouble.
  557. # (See https://github.com/pypa/pip/pull/462.)
  558. for dir in list(dirs):
  559. # Don't search in anything that looks like a virtualenv
  560. # environment
  561. if (
  562. os.path.lexists(
  563. os.path.join(root, dir, 'bin', 'python')
  564. ) or
  565. os.path.exists(
  566. os.path.join(
  567. root, dir, 'Scripts', 'Python.exe'
  568. )
  569. )):
  570. dirs.remove(dir)
  571. # Also don't search through tests
  572. elif dir == 'test' or dir == 'tests':
  573. dirs.remove(dir)
  574. filenames.extend([os.path.join(root, dir)
  575. for dir in dirs])
  576. filenames = [f for f in filenames if f.endswith('.egg-info')]
  577. if not filenames:
  578. raise InstallationError(
  579. "Files/directories not found in %s" % base
  580. )
  581. # if we have more than one match, we pick the toplevel one. This
  582. # can easily be the case if there is a dist folder which contains
  583. # an extracted tarball for testing purposes.
  584. if len(filenames) > 1:
  585. filenames.sort(
  586. key=lambda x: x.count(os.path.sep) +
  587. (os.path.altsep and x.count(os.path.altsep) or 0)
  588. )
  589. self._egg_info_path = os.path.join(base, filenames[0])
  590. return self._egg_info_path
  591. @property
  592. def metadata(self):
  593. if not hasattr(self, '_metadata'):
  594. self._metadata = get_metadata(self.get_dist())
  595. return self._metadata
  596. def get_dist(self):
  597. # type: () -> Distribution
  598. """Return a pkg_resources.Distribution for this requirement"""
  599. if self.metadata_directory:
  600. base_dir, distinfo = os.path.split(self.metadata_directory)
  601. metadata = pkg_resources.PathMetadata(
  602. base_dir, self.metadata_directory
  603. )
  604. dist_name = os.path.splitext(distinfo)[0]
  605. typ = pkg_resources.DistInfoDistribution
  606. else:
  607. egg_info = self.egg_info_path.rstrip(os.path.sep)
  608. base_dir = os.path.dirname(egg_info)
  609. metadata = pkg_resources.PathMetadata(base_dir, egg_info)
  610. dist_name = os.path.splitext(os.path.basename(egg_info))[0]
  611. # https://github.com/python/mypy/issues/1174
  612. typ = pkg_resources.Distribution # type: ignore
  613. return typ(
  614. base_dir,
  615. project_name=dist_name,
  616. metadata=metadata,
  617. )
  618. def assert_source_matches_version(self):
  619. # type: () -> None
  620. assert self.source_dir
  621. version = self.metadata['version']
  622. if self.req.specifier and version not in self.req.specifier:
  623. logger.warning(
  624. 'Requested %s, but installing version %s',
  625. self,
  626. version,
  627. )
  628. else:
  629. logger.debug(
  630. 'Source in %s has version %s, which satisfies requirement %s',
  631. display_path(self.source_dir),
  632. version,
  633. self,
  634. )
  635. # For both source distributions and editables
  636. def ensure_has_source_dir(self, parent_dir):
  637. # type: (str) -> str
  638. """Ensure that a source_dir is set.
  639. This will create a temporary build dir if the name of the requirement
  640. isn't known yet.
  641. :param parent_dir: The ideal pip parent_dir for the source_dir.
  642. Generally src_dir for editables and build_dir for sdists.
  643. :return: self.source_dir
  644. """
  645. if self.source_dir is None:
  646. self.source_dir = self.build_location(parent_dir)
  647. return self.source_dir
  648. # For editable installations
  649. def install_editable(
  650. self,
  651. install_options, # type: List[str]
  652. global_options=(), # type: Sequence[str]
  653. prefix=None # type: Optional[str]
  654. ):
  655. # type: (...) -> None
  656. logger.info('Running setup.py develop for %s', self.name)
  657. if self.isolated:
  658. global_options = list(global_options) + ["--no-user-cfg"]
  659. if prefix:
  660. prefix_param = ['--prefix={}'.format(prefix)]
  661. install_options = list(install_options) + prefix_param
  662. with indent_log():
  663. # FIXME: should we do --install-headers here too?
  664. with self.build_env:
  665. call_subprocess(
  666. [
  667. sys.executable,
  668. '-c',
  669. SETUPTOOLS_SHIM % self.setup_py
  670. ] +
  671. list(global_options) +
  672. ['develop', '--no-deps'] +
  673. list(install_options),
  674. cwd=self.setup_py_dir,
  675. show_stdout=False,
  676. )
  677. self.install_succeeded = True
  678. def update_editable(self, obtain=True):
  679. # type: (bool) -> None
  680. if not self.link:
  681. logger.debug(
  682. "Cannot update repository at %s; repository location is "
  683. "unknown",
  684. self.source_dir,
  685. )
  686. return
  687. assert self.editable
  688. assert self.source_dir
  689. if self.link.scheme == 'file':
  690. # Static paths don't get updated
  691. return
  692. assert '+' in self.link.url, "bad url: %r" % self.link.url
  693. if not self.update:
  694. return
  695. vc_type, url = self.link.url.split('+', 1)
  696. backend = vcs.get_backend(vc_type)
  697. if backend:
  698. vcs_backend = backend(self.link.url)
  699. if obtain:
  700. vcs_backend.obtain(self.source_dir)
  701. else:
  702. vcs_backend.export(self.source_dir)
  703. else:
  704. assert 0, (
  705. 'Unexpected version control type (in %s): %s'
  706. % (self.link, vc_type))
  707. # Top-level Actions
  708. def uninstall(self, auto_confirm=False, verbose=False,
  709. use_user_site=False):
  710. # type: (bool, bool, bool) -> Optional[UninstallPathSet]
  711. """
  712. Uninstall the distribution currently satisfying this requirement.
  713. Prompts before removing or modifying files unless
  714. ``auto_confirm`` is True.
  715. Refuses to delete or modify files outside of ``sys.prefix`` -
  716. thus uninstallation within a virtual environment can only
  717. modify that virtual environment, even if the virtualenv is
  718. linked to global site-packages.
  719. """
  720. if not self.check_if_exists(use_user_site):
  721. logger.warning("Skipping %s as it is not installed.", self.name)
  722. return None
  723. dist = self.satisfied_by or self.conflicts_with
  724. uninstalled_pathset = UninstallPathSet.from_dist(dist)
  725. uninstalled_pathset.remove(auto_confirm, verbose)
  726. return uninstalled_pathset
  727. def _clean_zip_name(self, name, prefix): # only used by archive.
  728. assert name.startswith(prefix + os.path.sep), (
  729. "name %r doesn't start with prefix %r" % (name, prefix)
  730. )
  731. name = name[len(prefix) + 1:]
  732. name = name.replace(os.path.sep, '/')
  733. return name
  734. def _get_archive_name(self, path, parentdir, rootdir):
  735. # type: (str, str, str) -> str
  736. path = os.path.join(parentdir, path)
  737. name = self._clean_zip_name(path, rootdir)
  738. return self.name + '/' + name
  739. # TODO: Investigate if this should be kept in InstallRequirement
  740. # Seems to be used only when VCS + downloads
  741. def archive(self, build_dir):
  742. # type: (str) -> None
  743. assert self.source_dir
  744. create_archive = True
  745. archive_name = '%s-%s.zip' % (self.name, self.metadata["version"])
  746. archive_path = os.path.join(build_dir, archive_name)
  747. if os.path.exists(archive_path):
  748. response = ask_path_exists(
  749. 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' %
  750. display_path(archive_path), ('i', 'w', 'b', 'a'))
  751. if response == 'i':
  752. create_archive = False
  753. elif response == 'w':
  754. logger.warning('Deleting %s', display_path(archive_path))
  755. os.remove(archive_path)
  756. elif response == 'b':
  757. dest_file = backup_dir(archive_path)
  758. logger.warning(
  759. 'Backing up %s to %s',
  760. display_path(archive_path),
  761. display_path(dest_file),
  762. )
  763. shutil.move(archive_path, dest_file)
  764. elif response == 'a':
  765. sys.exit(-1)
  766. if create_archive:
  767. zip = zipfile.ZipFile(
  768. archive_path, 'w', zipfile.ZIP_DEFLATED,
  769. allowZip64=True
  770. )
  771. dir = os.path.normcase(os.path.abspath(self.setup_py_dir))
  772. for dirpath, dirnames, filenames in os.walk(dir):
  773. if 'pip-egg-info' in dirnames:
  774. dirnames.remove('pip-egg-info')
  775. for dirname in dirnames:
  776. dir_arcname = self._get_archive_name(dirname,
  777. parentdir=dirpath,
  778. rootdir=dir)
  779. zipdir = zipfile.ZipInfo(dir_arcname + '/')
  780. zipdir.external_attr = 0x1ED << 16 # 0o755
  781. zip.writestr(zipdir, '')
  782. for filename in filenames:
  783. if filename == PIP_DELETE_MARKER_FILENAME:
  784. continue
  785. file_arcname = self._get_archive_name(filename,
  786. parentdir=dirpath,
  787. rootdir=dir)
  788. filename = os.path.join(dirpath, filename)
  789. zip.write(filename, file_arcname)
  790. zip.close()
  791. logger.info('Saved %s', display_path(archive_path))
  792. def install(
  793. self,
  794. install_options, # type: List[str]
  795. global_options=None, # type: Optional[Sequence[str]]
  796. root=None, # type: Optional[str]
  797. home=None, # type: Optional[str]
  798. prefix=None, # type: Optional[str]
  799. warn_script_location=True, # type: bool
  800. use_user_site=False, # type: bool
  801. pycompile=True # type: bool
  802. ):
  803. # type: (...) -> None
  804. global_options = global_options if global_options is not None else []
  805. if self.editable:
  806. self.install_editable(
  807. install_options, global_options, prefix=prefix,
  808. )
  809. return
  810. if self.is_wheel:
  811. version = wheel.wheel_version(self.source_dir)
  812. wheel.check_compatibility(version, self.name)
  813. self.move_wheel_files(
  814. self.source_dir, root=root, prefix=prefix, home=home,
  815. warn_script_location=warn_script_location,
  816. use_user_site=use_user_site, pycompile=pycompile,
  817. )
  818. self.install_succeeded = True
  819. return
  820. # Extend the list of global and install options passed on to
  821. # the setup.py call with the ones from the requirements file.
  822. # Options specified in requirements file override those
  823. # specified on the command line, since the last option given
  824. # to setup.py is the one that is used.
  825. global_options = list(global_options) + \
  826. self.options.get('global_options', [])
  827. install_options = list(install_options) + \
  828. self.options.get('install_options', [])
  829. if self.isolated:
  830. # https://github.com/python/mypy/issues/1174
  831. global_options = global_options + ["--no-user-cfg"] # type: ignore
  832. with TempDirectory(kind="record") as temp_dir:
  833. record_filename = os.path.join(temp_dir.path, 'install-record.txt')
  834. install_args = self.get_install_args(
  835. global_options, record_filename, root, prefix, pycompile,
  836. )
  837. msg = 'Running setup.py install for %s' % (self.name,)
  838. with open_spinner(msg) as spinner:
  839. with indent_log():
  840. with self.build_env:
  841. call_subprocess(
  842. install_args + install_options,
  843. cwd=self.setup_py_dir,
  844. show_stdout=False,
  845. spinner=spinner,
  846. )
  847. if not os.path.exists(record_filename):
  848. logger.debug('Record file %s not found', record_filename)
  849. return
  850. self.install_succeeded = True
  851. def prepend_root(path):
  852. if root is None or not os.path.isabs(path):
  853. return path
  854. else:
  855. return change_root(root, path)
  856. with open(record_filename) as f:
  857. for line in f:
  858. directory = os.path.dirname(line)
  859. if directory.endswith('.egg-info'):
  860. egg_info_dir = prepend_root(directory)
  861. break
  862. else:
  863. logger.warning(
  864. 'Could not find .egg-info directory in install record'
  865. ' for %s',
  866. self,
  867. )
  868. # FIXME: put the record somewhere
  869. # FIXME: should this be an error?
  870. return
  871. new_lines = []
  872. with open(record_filename) as f:
  873. for line in f:
  874. filename = line.strip()
  875. if os.path.isdir(filename):
  876. filename += os.path.sep
  877. new_lines.append(
  878. os.path.relpath(prepend_root(filename), egg_info_dir)
  879. )
  880. new_lines.sort()
  881. ensure_dir(egg_info_dir)
  882. inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
  883. with open(inst_files_path, 'w') as f:
  884. f.write('\n'.join(new_lines) + '\n')
  885. def get_install_args(
  886. self,
  887. global_options, # type: Sequence[str]
  888. record_filename, # type: str
  889. root, # type: Optional[str]
  890. prefix, # type: Optional[str]
  891. pycompile # type: bool
  892. ):
  893. # type: (...) -> List[str]
  894. install_args = [sys.executable, "-u"]
  895. install_args.append('-c')
  896. install_args.append(SETUPTOOLS_SHIM % self.setup_py)
  897. install_args += list(global_options) + \
  898. ['install', '--record', record_filename]
  899. install_args += ['--single-version-externally-managed']
  900. if root is not None:
  901. install_args += ['--root', root]
  902. if prefix is not None:
  903. install_args += ['--prefix', prefix]
  904. if pycompile:
  905. install_args += ["--compile"]
  906. else:
  907. install_args += ["--no-compile"]
  908. if running_under_virtualenv():
  909. py_ver_str = 'python' + sysconfig.get_python_version()
  910. install_args += ['--install-headers',
  911. os.path.join(sys.prefix, 'include', 'site',
  912. py_ver_str, self.name)]
  913. return install_args