<?xml version="1.0" encoding="UTF-8"?> | |||||
<module type="PYTHON_MODULE" version="4"> | |||||
<component name="NewModuleRootManager"> | |||||
<content url="file://$MODULE_DIR$" /> | |||||
<orderEntry type="inheritedJdk" /> | |||||
<orderEntry type="sourceFolder" forTests="false" /> | |||||
</component> | |||||
<component name="TestRunnerService"> | |||||
<option name="PROJECT_TEST_RUNNER" value="Unittests" /> | |||||
</component> | |||||
</module> |
<component name="InspectionProjectProfileManager"> | |||||
<settings> | |||||
<option name="USE_PROJECT_PROFILE" value="false" /> | |||||
<version value="1.0" /> | |||||
</settings> | |||||
</component> |
<?xml version="1.0" encoding="UTF-8"?> | |||||
<project version="4"> | |||||
<component name="JavaScriptSettings"> | |||||
<option name="languageLevel" value="ES6" /> | |||||
</component> | |||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" /> | |||||
</project> |
<?xml version="1.0" encoding="UTF-8"?> | |||||
<project version="4"> | |||||
<component name="ProjectModuleManager"> | |||||
<modules> | |||||
<module fileurl="file://$PROJECT_DIR$/.idea/Web_Engineering.iml" filepath="$PROJECT_DIR$/.idea/Web_Engineering.iml" /> | |||||
</modules> | |||||
</component> | |||||
</project> |
<?xml version="1.0" encoding="UTF-8"?> | |||||
<project version="4"> | |||||
<component name="ChangeListManager"> | |||||
<list default="true" id="8ccc1b48-23e0-4e30-98d2-f52b8f3df07a" name="Default Changelist" comment="" /> | |||||
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> | |||||
<option name="SHOW_DIALOG" value="false" /> | |||||
<option name="HIGHLIGHT_CONFLICTS" value="true" /> | |||||
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> | |||||
<option name="LAST_RESOLUTION" value="IGNORE" /> | |||||
</component> | |||||
<component name="FileTemplateManagerImpl"> | |||||
<option name="RECENT_TEMPLATES"> | |||||
<list> | |||||
<option value="Python Script" /> | |||||
</list> | |||||
</option> | |||||
</component> | |||||
<component name="ProjectId" id="1SEugOsGOB8hi2cwkLDRm5UoPiR" /> | |||||
<component name="PropertiesComponent"> | |||||
<property name="WebServerToolWindowFactoryState" value="false" /> | |||||
</component> | |||||
<component name="RunDashboard"> | |||||
<option name="ruleStates"> | |||||
<list> | |||||
<RuleState> | |||||
<option name="name" value="ConfigurationTypeDashboardGroupingRule" /> | |||||
</RuleState> | |||||
<RuleState> | |||||
<option name="name" value="StatusDashboardGroupingRule" /> | |||||
</RuleState> | |||||
</list> | |||||
</option> | |||||
</component> | |||||
<component name="RunManager" selected="Python.client_app"> | |||||
<configuration name="client_app" type="PythonConfigurationType" factoryName="Python"> | |||||
<module name="Web_Engineering" /> | |||||
<option name="INTERPRETER_OPTIONS" value="" /> | |||||
<option name="PARENT_ENVS" value="true" /> | |||||
<envs> | |||||
<env name="PYTHONUNBUFFERED" value="1" /> | |||||
</envs> | |||||
<option name="SDK_HOME" value="" /> | |||||
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" /> | |||||
<option name="IS_MODULE_SDK" value="true" /> | |||||
<option name="ADD_CONTENT_ROOTS" value="true" /> | |||||
<option name="ADD_SOURCE_ROOTS" value="true" /> | |||||
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> | |||||
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/client_app.py" /> | |||||
<option name="PARAMETERS" value="" /> | |||||
<option name="SHOW_COMMAND_LINE" value="false" /> | |||||
<option name="EMULATE_TERMINAL" value="false" /> | |||||
<option name="MODULE_MODE" value="false" /> | |||||
<option name="REDIRECT_INPUT" value="false" /> | |||||
<option name="INPUT_FILE" value="" /> | |||||
<method v="2" /> | |||||
</configuration> | |||||
<configuration name="server_app" type="PythonConfigurationType" factoryName="Python" temporary="true"> | |||||
<module name="Web_Engineering" /> | |||||
<option name="INTERPRETER_OPTIONS" value="" /> | |||||
<option name="PARENT_ENVS" value="true" /> | |||||
<envs> | |||||
<env name="PYTHONUNBUFFERED" value="1" /> | |||||
</envs> | |||||
<option name="SDK_HOME" value="" /> | |||||
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" /> | |||||
<option name="IS_MODULE_SDK" value="true" /> | |||||
<option name="ADD_CONTENT_ROOTS" value="true" /> | |||||
<option name="ADD_SOURCE_ROOTS" value="true" /> | |||||
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> | |||||
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/server_app.py" /> | |||||
<option name="PARAMETERS" value="" /> | |||||
<option name="SHOW_COMMAND_LINE" value="false" /> | |||||
<option name="EMULATE_TERMINAL" value="false" /> | |||||
<option name="MODULE_MODE" value="false" /> | |||||
<option name="REDIRECT_INPUT" value="false" /> | |||||
<option name="INPUT_FILE" value="" /> | |||||
<method v="2" /> | |||||
</configuration> | |||||
<recent_temporary> | |||||
<list> | |||||
<item itemvalue="Python.server_app" /> | |||||
</list> | |||||
</recent_temporary> | |||||
</component> | |||||
<component name="SvnConfiguration"> | |||||
<configuration /> | |||||
</component> | |||||
<component name="TaskManager"> | |||||
<task active="true" id="Default" summary="Default task"> | |||||
<changelist id="8ccc1b48-23e0-4e30-98d2-f52b8f3df07a" name="Default Changelist" comment="" /> | |||||
<created>1571141186964</created> | |||||
<option name="number" value="Default" /> | |||||
<option name="presentableId" value="Default" /> | |||||
<updated>1571141186964</updated> | |||||
<workItem from="1571141189448" duration="1240000" /> | |||||
</task> | |||||
<servers /> | |||||
</component> | |||||
<component name="TypeScriptGeneratedFilesManager"> | |||||
<option name="version" value="1" /> | |||||
</component> | |||||
<component name="com.intellij.coverage.CoverageDataManagerImpl"> | |||||
<SUITE FILE_PATH="coverage/Web_Engineering$client_app.coverage" NAME="client_app Coverage Results" MODIFIED="1571142046824" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" /> | |||||
<SUITE FILE_PATH="coverage/Web_Engineering$server_app.coverage" NAME="server_app Coverage Results" MODIFIED="1571142041364" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" /> | |||||
</component> | |||||
</project> |
import socket | |||||
s = socket.socket() | |||||
host = socket.gethostname() | |||||
port = 61233 | |||||
eingabe = " " | |||||
while eingabe != "STOP": | |||||
eingabe = input("Bitte String eingeben ") | |||||
s.connect((host, port)) | |||||
s.send(eingabe.encode("utf-8")) | |||||
bytes = s.recv(1024) | |||||
print(bytes.decode("utf-8")) | |||||
s.close() |
import socket | |||||
s = socket.socket() | |||||
host = socket.gethostname() | |||||
port = 61233 | |||||
s.bind((host, port)) | |||||
s.listen() | |||||
while True: | |||||
(connection, addr) = s.accept() | |||||
print("Verbindung von ", addr) | |||||
data = connection.recv(1024) | |||||
data = data.decode("utf-8") | |||||
#print("Data = %s" % (data)) | |||||
data = data.upper() | |||||
#data = "hi" | |||||
connection.send(data.encode("utf-8")) | |||||
connection.close() |
# This file must be used with "source bin/activate" *from bash* | |||||
# you cannot run it directly | |||||
deactivate () { | |||||
# reset old environment variables | |||||
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then | |||||
PATH="${_OLD_VIRTUAL_PATH:-}" | |||||
export PATH | |||||
unset _OLD_VIRTUAL_PATH | |||||
fi | |||||
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then | |||||
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" | |||||
export PYTHONHOME | |||||
unset _OLD_VIRTUAL_PYTHONHOME | |||||
fi | |||||
# This should detect bash and zsh, which have a hash command that must | |||||
# be called to get it to forget past commands. Without forgetting | |||||
# past commands the $PATH changes we made may not be respected | |||||
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then | |||||
hash -r | |||||
fi | |||||
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then | |||||
PS1="${_OLD_VIRTUAL_PS1:-}" | |||||
export PS1 | |||||
unset _OLD_VIRTUAL_PS1 | |||||
fi | |||||
unset VIRTUAL_ENV | |||||
if [ ! "$1" = "nondestructive" ] ; then | |||||
# Self destruct! | |||||
unset -f deactivate | |||||
fi | |||||
} | |||||
# unset irrelevant variables | |||||
deactivate nondestructive | |||||
VIRTUAL_ENV="/Users/irakha63255/Desktop/Neues_Projekt/venv" | |||||
export VIRTUAL_ENV | |||||
_OLD_VIRTUAL_PATH="$PATH" | |||||
PATH="$VIRTUAL_ENV/bin:$PATH" | |||||
export PATH | |||||
# unset PYTHONHOME if set | |||||
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) | |||||
# could use `if (set -u; : $PYTHONHOME) ;` in bash | |||||
if [ -n "${PYTHONHOME:-}" ] ; then | |||||
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" | |||||
unset PYTHONHOME | |||||
fi | |||||
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then | |||||
_OLD_VIRTUAL_PS1="${PS1:-}" | |||||
if [ "x(venv) " != x ] ; then | |||||
PS1="(venv) ${PS1:-}" | |||||
else | |||||
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then | |||||
# special case for Aspen magic directories | |||||
# see http://www.zetadev.com/software/aspen/ | |||||
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" | |||||
else | |||||
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" | |||||
fi | |||||
fi | |||||
export PS1 | |||||
fi | |||||
# This should detect bash and zsh, which have a hash command that must | |||||
# be called to get it to forget past commands. Without forgetting | |||||
# past commands the $PATH changes we made may not be respected | |||||
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then | |||||
hash -r | |||||
fi |
# This file must be used with "source bin/activate.csh" *from csh*. | |||||
# You cannot run it directly. | |||||
# Created by Davide Di Blasi <davidedb@gmail.com>. | |||||
# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com> | |||||
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' | |||||
# Unset irrelevant variables. | |||||
deactivate nondestructive | |||||
setenv VIRTUAL_ENV "/Users/irakha63255/Desktop/Neues_Projekt/venv" | |||||
set _OLD_VIRTUAL_PATH="$PATH" | |||||
setenv PATH "$VIRTUAL_ENV/bin:$PATH" | |||||
set _OLD_VIRTUAL_PROMPT="$prompt" | |||||
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then | |||||
if ("venv" != "") then | |||||
set env_name = "venv" | |||||
else | |||||
if (`basename "VIRTUAL_ENV"` == "__") then | |||||
# special case for Aspen magic directories | |||||
# see http://www.zetadev.com/software/aspen/ | |||||
set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` | |||||
else | |||||
set env_name = `basename "$VIRTUAL_ENV"` | |||||
endif | |||||
endif | |||||
set prompt = "[$env_name] $prompt" | |||||
unset env_name | |||||
endif | |||||
alias pydoc python -m pydoc | |||||
rehash |
# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) | |||||
# you cannot run it directly | |||||
function deactivate -d "Exit virtualenv and return to normal shell environment" | |||||
# reset old environment variables | |||||
if test -n "$_OLD_VIRTUAL_PATH" | |||||
set -gx PATH $_OLD_VIRTUAL_PATH | |||||
set -e _OLD_VIRTUAL_PATH | |||||
end | |||||
if test -n "$_OLD_VIRTUAL_PYTHONHOME" | |||||
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME | |||||
set -e _OLD_VIRTUAL_PYTHONHOME | |||||
end | |||||
if test -n "$_OLD_FISH_PROMPT_OVERRIDE" | |||||
functions -e fish_prompt | |||||
set -e _OLD_FISH_PROMPT_OVERRIDE | |||||
functions -c _old_fish_prompt fish_prompt | |||||
functions -e _old_fish_prompt | |||||
end | |||||
set -e VIRTUAL_ENV | |||||
if test "$argv[1]" != "nondestructive" | |||||
# Self destruct! | |||||
functions -e deactivate | |||||
end | |||||
end | |||||
# unset irrelevant variables | |||||
deactivate nondestructive | |||||
set -gx VIRTUAL_ENV "/Users/irakha63255/Desktop/Neues_Projekt/venv" | |||||
set -gx _OLD_VIRTUAL_PATH $PATH | |||||
set -gx PATH "$VIRTUAL_ENV/bin" $PATH | |||||
# unset PYTHONHOME if set | |||||
if set -q PYTHONHOME | |||||
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME | |||||
set -e PYTHONHOME | |||||
end | |||||
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" | |||||
# fish uses a function instead of an env var to generate the prompt. | |||||
# save the current fish_prompt function as the function _old_fish_prompt | |||||
functions -c fish_prompt _old_fish_prompt | |||||
# with the original prompt function renamed, we can override with our own. | |||||
function fish_prompt | |||||
# Save the return status of the last command | |||||
set -l old_status $status | |||||
# Prompt override? | |||||
if test -n "(venv) " | |||||
printf "%s%s" "(venv) " (set_color normal) | |||||
else | |||||
# ...Otherwise, prepend env | |||||
set -l _checkbase (basename "$VIRTUAL_ENV") | |||||
if test $_checkbase = "__" | |||||
# special case for Aspen magic directories | |||||
# see http://www.zetadev.com/software/aspen/ | |||||
printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) | |||||
else | |||||
printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) | |||||
end | |||||
end | |||||
# Restore the return status of the previous command. | |||||
echo "exit $old_status" | . | |||||
_old_fish_prompt | |||||
end | |||||
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" | |||||
end |
#!/Users/irakha63255/Desktop/Neues_Projekt/venv/bin/python | |||||
# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==39.1.0','console_scripts','easy_install' | |||||
__requires__ = 'setuptools==39.1.0' | |||||
import re | |||||
import sys | |||||
from pkg_resources import load_entry_point | |||||
if __name__ == '__main__': | |||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) | |||||
sys.exit( | |||||
load_entry_point('setuptools==39.1.0', 'console_scripts', 'easy_install')() | |||||
) |
#!/Users/irakha63255/Desktop/Neues_Projekt/venv/bin/python | |||||
# EASY-INSTALL-ENTRY-SCRIPT: 'setuptools==39.1.0','console_scripts','easy_install-3.7' | |||||
__requires__ = 'setuptools==39.1.0' | |||||
import re | |||||
import sys | |||||
from pkg_resources import load_entry_point | |||||
if __name__ == '__main__': | |||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) | |||||
sys.exit( | |||||
load_entry_point('setuptools==39.1.0', 'console_scripts', 'easy_install-3.7')() | |||||
) |
#!/Users/irakha63255/Desktop/Neues_Projekt/venv/bin/python | |||||
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip' | |||||
__requires__ = 'pip==10.0.1' | |||||
import re | |||||
import sys | |||||
from pkg_resources import load_entry_point | |||||
if __name__ == '__main__': | |||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) | |||||
sys.exit( | |||||
load_entry_point('pip==10.0.1', 'console_scripts', 'pip')() | |||||
) |
#!/Users/irakha63255/Desktop/Neues_Projekt/venv/bin/python | |||||
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip3' | |||||
__requires__ = 'pip==10.0.1' | |||||
import re | |||||
import sys | |||||
from pkg_resources import load_entry_point | |||||
if __name__ == '__main__': | |||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) | |||||
sys.exit( | |||||
load_entry_point('pip==10.0.1', 'console_scripts', 'pip3')() | |||||
) |
#!/Users/irakha63255/Desktop/Neues_Projekt/venv/bin/python | |||||
# EASY-INSTALL-ENTRY-SCRIPT: 'pip==10.0.1','console_scripts','pip3.7' | |||||
__requires__ = 'pip==10.0.1' | |||||
import re | |||||
import sys | |||||
from pkg_resources import load_entry_point | |||||
if __name__ == '__main__': | |||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) | |||||
sys.exit( | |||||
load_entry_point('pip==10.0.1', 'console_scripts', 'pip3.7')() | |||||
) |
./setuptools-39.1.0-py3.7.egg | |||||
./pip-10.0.1-py3.7.egg |
Metadata-Version: 2.1 | |||||
Name: pip | |||||
Version: 10.0.1 | |||||
Summary: The PyPA recommended tool for installing Python packages. | |||||
Home-page: https://pip.pypa.io/ | |||||
Author: The pip developers | |||||
Author-email: python-virtualenv@groups.google.com | |||||
License: MIT | |||||
Description: pip | |||||
=== | |||||
The `PyPA recommended`_ tool for installing Python packages. | |||||
.. image:: https://img.shields.io/pypi/v/pip.svg | |||||
:target: https://pypi.org/project/pip/ | |||||
.. image:: https://img.shields.io/travis/pypa/pip/master.svg | |||||
:target: http://travis-ci.org/pypa/pip | |||||
.. image:: https://img.shields.io/appveyor/ci/pypa/pip.svg | |||||
:target: https://ci.appveyor.com/project/pypa/pip/history | |||||
.. image:: https://readthedocs.org/projects/pip/badge/?version=latest | |||||
:target: https://pip.pypa.io/en/latest | |||||
* `Installation`_ | |||||
* `Documentation`_ | |||||
* `Changelog`_ | |||||
* `GitHub Page`_ | |||||
* `Issue Tracking`_ | |||||
* `User mailing list`_ | |||||
* `Dev mailing list`_ | |||||
* User IRC: #pypa on Freenode. | |||||
* Dev IRC: #pypa-dev on Freenode. | |||||
Code of Conduct | |||||
--------------- | |||||
Everyone interacting in the pip project's codebases, issue trackers, chat | |||||
rooms and mailing lists is expected to follow the `PyPA Code of Conduct`_. | |||||
.. _PyPA recommended: https://packaging.python.org/en/latest/current/ | |||||
.. _Installation: https://pip.pypa.io/en/stable/installing.html | |||||
.. _Documentation: https://pip.pypa.io/en/stable/ | |||||
.. _Changelog: https://pip.pypa.io/en/stable/news.html | |||||
.. _GitHub Page: https://github.com/pypa/pip | |||||
.. _Issue Tracking: https://github.com/pypa/pip/issues | |||||
.. _User mailing list: http://groups.google.com/group/python-virtualenv | |||||
.. _Dev mailing list: http://groups.google.com/group/pypa-dev | |||||
.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ | |||||
Keywords: easy_install distutils setuptools egg virtualenv | |||||
Platform: UNKNOWN | |||||
Classifier: Development Status :: 5 - Production/Stable | |||||
Classifier: Intended Audience :: Developers | |||||
Classifier: License :: OSI Approved :: MIT License | |||||
Classifier: Topic :: Software Development :: Build Tools | |||||
Classifier: Programming Language :: Python | |||||
Classifier: Programming Language :: Python :: 2 | |||||
Classifier: Programming Language :: Python :: 2.7 | |||||
Classifier: Programming Language :: Python :: 3 | |||||
Classifier: Programming Language :: Python :: 3.3 | |||||
Classifier: Programming Language :: Python :: 3.4 | |||||
Classifier: Programming Language :: Python :: 3.5 | |||||
Classifier: Programming Language :: Python :: 3.6 | |||||
Classifier: Programming Language :: Python :: Implementation :: CPython | |||||
Classifier: Programming Language :: Python :: Implementation :: PyPy | |||||
Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.* | |||||
Provides-Extra: testing |
AUTHORS.txt | |||||
LICENSE.txt | |||||
MANIFEST.in | |||||
NEWS.rst | |||||
README.rst | |||||
pyproject.toml | |||||
setup.cfg | |||||
setup.py | |||||
docs/Makefile | |||||
docs/__init__.py | |||||
docs/conf.py | |||||
docs/configuration.rst | |||||
docs/cookbook.rst | |||||
docs/development.rst | |||||
docs/docutils.conf | |||||
docs/index.rst | |||||
docs/installing.rst | |||||
docs/logic.rst | |||||
docs/make.bat | |||||
docs/news.rst | |||||
docs/pipext.py | |||||
docs/quickstart.rst | |||||
docs/usage.rst | |||||
docs/user_guide.rst | |||||
docs/man/pip.rst | |||||
docs/man/commands/check.rst | |||||
docs/man/commands/config.rst | |||||
docs/man/commands/download.rst | |||||
docs/man/commands/freeze.rst | |||||
docs/man/commands/hash.rst | |||||
docs/man/commands/help.rst | |||||
docs/man/commands/install.rst | |||||
docs/man/commands/list.rst | |||||
docs/man/commands/search.rst | |||||
docs/man/commands/show.rst | |||||
docs/man/commands/uninstall.rst | |||||
docs/man/commands/wheel.rst | |||||
docs/reference/index.rst | |||||
docs/reference/pip.rst | |||||
docs/reference/pip_check.rst | |||||
docs/reference/pip_config.rst | |||||
docs/reference/pip_download.rst | |||||
docs/reference/pip_freeze.rst | |||||
docs/reference/pip_hash.rst | |||||
docs/reference/pip_install.rst | |||||
docs/reference/pip_list.rst | |||||
docs/reference/pip_search.rst | |||||
docs/reference/pip_show.rst | |||||
docs/reference/pip_uninstall.rst | |||||
docs/reference/pip_wheel.rst | |||||
src/pip/__init__.py | |||||
src/pip/__main__.py | |||||
src/pip.egg-info/PKG-INFO | |||||
src/pip.egg-info/SOURCES.txt | |||||
src/pip.egg-info/dependency_links.txt | |||||
src/pip.egg-info/entry_points.txt | |||||
src/pip.egg-info/not-zip-safe | |||||
src/pip.egg-info/requires.txt | |||||
src/pip.egg-info/top_level.txt | |||||
src/pip/_internal/__init__.py | |||||
src/pip/_internal/basecommand.py | |||||
src/pip/_internal/baseparser.py | |||||
src/pip/_internal/build_env.py | |||||
src/pip/_internal/cache.py | |||||
src/pip/_internal/cmdoptions.py | |||||
src/pip/_internal/compat.py | |||||
src/pip/_internal/configuration.py | |||||
src/pip/_internal/download.py | |||||
src/pip/_internal/exceptions.py | |||||
src/pip/_internal/index.py | |||||
src/pip/_internal/locations.py | |||||
src/pip/_internal/pep425tags.py | |||||
src/pip/_internal/resolve.py | |||||
src/pip/_internal/status_codes.py | |||||
src/pip/_internal/wheel.py | |||||
src/pip/_internal/commands/__init__.py | |||||
src/pip/_internal/commands/check.py | |||||
src/pip/_internal/commands/completion.py | |||||
src/pip/_internal/commands/configuration.py | |||||
src/pip/_internal/commands/download.py | |||||
src/pip/_internal/commands/freeze.py | |||||
src/pip/_internal/commands/hash.py | |||||
src/pip/_internal/commands/help.py | |||||
src/pip/_internal/commands/install.py | |||||
src/pip/_internal/commands/list.py | |||||
src/pip/_internal/commands/search.py | |||||
src/pip/_internal/commands/show.py | |||||
src/pip/_internal/commands/uninstall.py | |||||
src/pip/_internal/commands/wheel.py | |||||
src/pip/_internal/models/__init__.py | |||||
src/pip/_internal/models/index.py | |||||
src/pip/_internal/operations/__init__.py | |||||
src/pip/_internal/operations/check.py | |||||
src/pip/_internal/operations/freeze.py | |||||
src/pip/_internal/operations/prepare.py | |||||
src/pip/_internal/req/__init__.py | |||||
src/pip/_internal/req/req_file.py | |||||
src/pip/_internal/req/req_install.py | |||||
src/pip/_internal/req/req_set.py | |||||
src/pip/_internal/req/req_uninstall.py | |||||
src/pip/_internal/utils/__init__.py | |||||
src/pip/_internal/utils/appdirs.py | |||||
src/pip/_internal/utils/deprecation.py | |||||
src/pip/_internal/utils/encoding.py | |||||
src/pip/_internal/utils/filesystem.py | |||||
src/pip/_internal/utils/glibc.py | |||||
src/pip/_internal/utils/hashes.py | |||||
src/pip/_internal/utils/logging.py | |||||
src/pip/_internal/utils/misc.py | |||||
src/pip/_internal/utils/outdated.py | |||||
src/pip/_internal/utils/packaging.py | |||||
src/pip/_internal/utils/setuptools_build.py | |||||
src/pip/_internal/utils/temp_dir.py | |||||
src/pip/_internal/utils/typing.py | |||||
src/pip/_internal/utils/ui.py | |||||
src/pip/_internal/vcs/__init__.py | |||||
src/pip/_internal/vcs/bazaar.py | |||||
src/pip/_internal/vcs/git.py | |||||
src/pip/_internal/vcs/mercurial.py | |||||
src/pip/_internal/vcs/subversion.py | |||||
src/pip/_vendor/README.rst | |||||
src/pip/_vendor/__init__.py | |||||
src/pip/_vendor/appdirs.py | |||||
src/pip/_vendor/distro.py | |||||
src/pip/_vendor/ipaddress.py | |||||
src/pip/_vendor/pyparsing.py | |||||
src/pip/_vendor/retrying.py | |||||
src/pip/_vendor/six.py | |||||
src/pip/_vendor/vendor.txt | |||||
src/pip/_vendor/cachecontrol/__init__.py | |||||
src/pip/_vendor/cachecontrol/_cmd.py | |||||
src/pip/_vendor/cachecontrol/adapter.py | |||||
src/pip/_vendor/cachecontrol/cache.py | |||||
src/pip/_vendor/cachecontrol/compat.py | |||||
src/pip/_vendor/cachecontrol/controller.py | |||||
src/pip/_vendor/cachecontrol/filewrapper.py | |||||
src/pip/_vendor/cachecontrol/heuristics.py | |||||
src/pip/_vendor/cachecontrol/serialize.py | |||||
src/pip/_vendor/cachecontrol/wrapper.py | |||||
src/pip/_vendor/cachecontrol/caches/__init__.py | |||||
src/pip/_vendor/cachecontrol/caches/file_cache.py | |||||
src/pip/_vendor/cachecontrol/caches/redis_cache.py | |||||
src/pip/_vendor/certifi/__init__.py | |||||
src/pip/_vendor/certifi/__main__.py | |||||
src/pip/_vendor/certifi/cacert.pem | |||||
src/pip/_vendor/certifi/core.py | |||||
src/pip/_vendor/chardet/__init__.py | |||||
src/pip/_vendor/chardet/big5freq.py | |||||
src/pip/_vendor/chardet/big5prober.py | |||||
src/pip/_vendor/chardet/chardistribution.py | |||||
src/pip/_vendor/chardet/charsetgroupprober.py | |||||
src/pip/_vendor/chardet/charsetprober.py | |||||
src/pip/_vendor/chardet/codingstatemachine.py | |||||
src/pip/_vendor/chardet/compat.py | |||||
src/pip/_vendor/chardet/cp949prober.py | |||||
src/pip/_vendor/chardet/enums.py | |||||
src/pip/_vendor/chardet/escprober.py | |||||
src/pip/_vendor/chardet/escsm.py | |||||
src/pip/_vendor/chardet/eucjpprober.py | |||||
src/pip/_vendor/chardet/euckrfreq.py | |||||
src/pip/_vendor/chardet/euckrprober.py | |||||
src/pip/_vendor/chardet/euctwfreq.py | |||||
src/pip/_vendor/chardet/euctwprober.py | |||||
src/pip/_vendor/chardet/gb2312freq.py | |||||
src/pip/_vendor/chardet/gb2312prober.py | |||||
src/pip/_vendor/chardet/hebrewprober.py | |||||
src/pip/_vendor/chardet/jisfreq.py | |||||
src/pip/_vendor/chardet/jpcntx.py | |||||
src/pip/_vendor/chardet/langbulgarianmodel.py | |||||
src/pip/_vendor/chardet/langcyrillicmodel.py | |||||
src/pip/_vendor/chardet/langgreekmodel.py | |||||
src/pip/_vendor/chardet/langhebrewmodel.py | |||||
src/pip/_vendor/chardet/langhungarianmodel.py | |||||
src/pip/_vendor/chardet/langthaimodel.py | |||||
src/pip/_vendor/chardet/langturkishmodel.py | |||||
src/pip/_vendor/chardet/latin1prober.py | |||||
src/pip/_vendor/chardet/mbcharsetprober.py | |||||
src/pip/_vendor/chardet/mbcsgroupprober.py | |||||
src/pip/_vendor/chardet/mbcssm.py | |||||
src/pip/_vendor/chardet/sbcharsetprober.py | |||||
src/pip/_vendor/chardet/sbcsgroupprober.py | |||||
src/pip/_vendor/chardet/sjisprober.py | |||||
src/pip/_vendor/chardet/universaldetector.py | |||||
src/pip/_vendor/chardet/utf8prober.py | |||||
src/pip/_vendor/chardet/version.py | |||||
src/pip/_vendor/chardet/cli/__init__.py | |||||
src/pip/_vendor/chardet/cli/chardetect.py | |||||
src/pip/_vendor/colorama/__init__.py | |||||
src/pip/_vendor/colorama/ansi.py | |||||
src/pip/_vendor/colorama/ansitowin32.py | |||||
src/pip/_vendor/colorama/initialise.py | |||||
src/pip/_vendor/colorama/win32.py | |||||
src/pip/_vendor/colorama/winterm.py | |||||
src/pip/_vendor/distlib/__init__.py | |||||
src/pip/_vendor/distlib/compat.py | |||||
src/pip/_vendor/distlib/database.py | |||||
src/pip/_vendor/distlib/index.py | |||||
src/pip/_vendor/distlib/locators.py | |||||
src/pip/_vendor/distlib/manifest.py | |||||
src/pip/_vendor/distlib/markers.py | |||||
src/pip/_vendor/distlib/metadata.py | |||||
src/pip/_vendor/distlib/resources.py | |||||
src/pip/_vendor/distlib/scripts.py | |||||
src/pip/_vendor/distlib/t32.exe | |||||
src/pip/_vendor/distlib/t64.exe | |||||
src/pip/_vendor/distlib/util.py | |||||
src/pip/_vendor/distlib/version.py | |||||
src/pip/_vendor/distlib/w32.exe | |||||
src/pip/_vendor/distlib/w64.exe | |||||
src/pip/_vendor/distlib/wheel.py | |||||
src/pip/_vendor/distlib/_backport/__init__.py | |||||
src/pip/_vendor/distlib/_backport/misc.py | |||||
src/pip/_vendor/distlib/_backport/shutil.py | |||||
src/pip/_vendor/distlib/_backport/sysconfig.cfg | |||||
src/pip/_vendor/distlib/_backport/sysconfig.py | |||||
src/pip/_vendor/distlib/_backport/tarfile.py | |||||
src/pip/_vendor/html5lib/__init__.py | |||||
src/pip/_vendor/html5lib/_ihatexml.py | |||||
src/pip/_vendor/html5lib/_inputstream.py | |||||
src/pip/_vendor/html5lib/_tokenizer.py | |||||
src/pip/_vendor/html5lib/_utils.py | |||||
src/pip/_vendor/html5lib/constants.py | |||||
src/pip/_vendor/html5lib/html5parser.py | |||||
src/pip/_vendor/html5lib/serializer.py | |||||
src/pip/_vendor/html5lib/_trie/__init__.py | |||||
src/pip/_vendor/html5lib/_trie/_base.py | |||||
src/pip/_vendor/html5lib/_trie/datrie.py | |||||
src/pip/_vendor/html5lib/_trie/py.py | |||||
src/pip/_vendor/html5lib/filters/__init__.py | |||||
src/pip/_vendor/html5lib/filters/alphabeticalattributes.py | |||||
src/pip/_vendor/html5lib/filters/base.py | |||||
src/pip/_vendor/html5lib/filters/inject_meta_charset.py | |||||
src/pip/_vendor/html5lib/filters/lint.py | |||||
src/pip/_vendor/html5lib/filters/optionaltags.py | |||||
src/pip/_vendor/html5lib/filters/sanitizer.py | |||||
src/pip/_vendor/html5lib/filters/whitespace.py | |||||
src/pip/_vendor/html5lib/treeadapters/__init__.py | |||||
src/pip/_vendor/html5lib/treeadapters/genshi.py | |||||
src/pip/_vendor/html5lib/treeadapters/sax.py | |||||
src/pip/_vendor/html5lib/treebuilders/__init__.py | |||||
src/pip/_vendor/html5lib/treebuilders/base.py | |||||
src/pip/_vendor/html5lib/treebuilders/dom.py | |||||
src/pip/_vendor/html5lib/treebuilders/etree.py | |||||
src/pip/_vendor/html5lib/treebuilders/etree_lxml.py | |||||
src/pip/_vendor/html5lib/treewalkers/__init__.py | |||||
src/pip/_vendor/html5lib/treewalkers/base.py | |||||
src/pip/_vendor/html5lib/treewalkers/dom.py | |||||
src/pip/_vendor/html5lib/treewalkers/etree.py | |||||
src/pip/_vendor/html5lib/treewalkers/etree_lxml.py | |||||
src/pip/_vendor/html5lib/treewalkers/genshi.py | |||||
src/pip/_vendor/idna/__init__.py | |||||
src/pip/_vendor/idna/codec.py | |||||
src/pip/_vendor/idna/compat.py | |||||
src/pip/_vendor/idna/core.py | |||||
src/pip/_vendor/idna/idnadata.py | |||||
src/pip/_vendor/idna/intranges.py | |||||
src/pip/_vendor/idna/package_data.py | |||||
src/pip/_vendor/idna/uts46data.py | |||||
src/pip/_vendor/lockfile/__init__.py | |||||
src/pip/_vendor/lockfile/linklockfile.py | |||||
src/pip/_vendor/lockfile/mkdirlockfile.py | |||||
src/pip/_vendor/lockfile/pidlockfile.py | |||||
src/pip/_vendor/lockfile/sqlitelockfile.py | |||||
src/pip/_vendor/lockfile/symlinklockfile.py | |||||
src/pip/_vendor/msgpack/__init__.py | |||||
src/pip/_vendor/msgpack/_version.py | |||||
src/pip/_vendor/msgpack/exceptions.py | |||||
src/pip/_vendor/msgpack/fallback.py | |||||
src/pip/_vendor/packaging/__about__.py | |||||
src/pip/_vendor/packaging/__init__.py | |||||
src/pip/_vendor/packaging/_compat.py | |||||
src/pip/_vendor/packaging/_structures.py | |||||
src/pip/_vendor/packaging/markers.py | |||||
src/pip/_vendor/packaging/requirements.py | |||||
src/pip/_vendor/packaging/specifiers.py | |||||
src/pip/_vendor/packaging/utils.py | |||||
src/pip/_vendor/packaging/version.py | |||||
src/pip/_vendor/pkg_resources/__init__.py | |||||
src/pip/_vendor/pkg_resources/py31compat.py | |||||
src/pip/_vendor/progress/__init__.py | |||||
src/pip/_vendor/progress/bar.py | |||||
src/pip/_vendor/progress/counter.py | |||||
src/pip/_vendor/progress/helpers.py | |||||
src/pip/_vendor/progress/spinner.py | |||||
src/pip/_vendor/pytoml/__init__.py | |||||
src/pip/_vendor/pytoml/core.py | |||||
src/pip/_vendor/pytoml/parser.py | |||||
src/pip/_vendor/pytoml/writer.py | |||||
src/pip/_vendor/requests/__init__.py | |||||
src/pip/_vendor/requests/__version__.py | |||||
src/pip/_vendor/requests/_internal_utils.py | |||||
src/pip/_vendor/requests/adapters.py | |||||
src/pip/_vendor/requests/api.py | |||||
src/pip/_vendor/requests/auth.py | |||||
src/pip/_vendor/requests/certs.py | |||||
src/pip/_vendor/requests/compat.py | |||||
src/pip/_vendor/requests/cookies.py | |||||
src/pip/_vendor/requests/exceptions.py | |||||
src/pip/_vendor/requests/help.py | |||||
src/pip/_vendor/requests/hooks.py | |||||
src/pip/_vendor/requests/models.py | |||||
src/pip/_vendor/requests/packages.py | |||||
src/pip/_vendor/requests/sessions.py | |||||
src/pip/_vendor/requests/status_codes.py | |||||
src/pip/_vendor/requests/structures.py | |||||
src/pip/_vendor/requests/utils.py | |||||
src/pip/_vendor/urllib3/__init__.py | |||||
src/pip/_vendor/urllib3/_collections.py | |||||
src/pip/_vendor/urllib3/connection.py | |||||
src/pip/_vendor/urllib3/connectionpool.py | |||||
src/pip/_vendor/urllib3/exceptions.py | |||||
src/pip/_vendor/urllib3/fields.py | |||||
src/pip/_vendor/urllib3/filepost.py | |||||
src/pip/_vendor/urllib3/poolmanager.py | |||||
src/pip/_vendor/urllib3/request.py | |||||
src/pip/_vendor/urllib3/response.py | |||||
src/pip/_vendor/urllib3/contrib/__init__.py | |||||
src/pip/_vendor/urllib3/contrib/appengine.py | |||||
src/pip/_vendor/urllib3/contrib/ntlmpool.py | |||||
src/pip/_vendor/urllib3/contrib/pyopenssl.py | |||||
src/pip/_vendor/urllib3/contrib/securetransport.py | |||||
src/pip/_vendor/urllib3/contrib/socks.py | |||||
src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py | |||||
src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py | |||||
src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py | |||||
src/pip/_vendor/urllib3/packages/__init__.py | |||||
src/pip/_vendor/urllib3/packages/ordered_dict.py | |||||
src/pip/_vendor/urllib3/packages/six.py | |||||
src/pip/_vendor/urllib3/packages/backports/__init__.py | |||||
src/pip/_vendor/urllib3/packages/backports/makefile.py | |||||
src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py | |||||
src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py | |||||
src/pip/_vendor/urllib3/util/__init__.py | |||||
src/pip/_vendor/urllib3/util/connection.py | |||||
src/pip/_vendor/urllib3/util/request.py | |||||
src/pip/_vendor/urllib3/util/response.py | |||||
src/pip/_vendor/urllib3/util/retry.py | |||||
src/pip/_vendor/urllib3/util/selectors.py | |||||
src/pip/_vendor/urllib3/util/ssl_.py | |||||
src/pip/_vendor/urllib3/util/timeout.py | |||||
src/pip/_vendor/urllib3/util/url.py | |||||
src/pip/_vendor/urllib3/util/wait.py | |||||
src/pip/_vendor/webencodings/__init__.py | |||||
src/pip/_vendor/webencodings/labels.py | |||||
src/pip/_vendor/webencodings/mklabels.py | |||||
src/pip/_vendor/webencodings/tests.py | |||||
src/pip/_vendor/webencodings/x_user_defined.py |
[console_scripts] | |||||
pip = pip._internal:main | |||||
pip3 = pip._internal:main | |||||
pip3.7 = pip._internal:main | |||||
[testing] | |||||
pytest | |||||
mock | |||||
pretend | |||||
scripttest>=1.3 | |||||
virtualenv>=1.10 | |||||
freezegun |
pip |
__version__ = "10.0.1" |
from __future__ import absolute_import | |||||
import os | |||||
import sys | |||||
# If we are running from a wheel, add the wheel to sys.path | |||||
# This allows the usage python pip-*.whl/pip install pip-*.whl | |||||
if __package__ == '': | |||||
# __file__ is pip-*.whl/pip/__main__.py | |||||
# first dirname call strips of '/__main__.py', second strips off '/pip' | |||||
# Resulting path is the name of the wheel itself | |||||
# Add that to sys.path so we can import pip | |||||
path = os.path.dirname(os.path.dirname(__file__)) | |||||
sys.path.insert(0, path) | |||||
from pip._internal import main as _main # noqa | |||||
if __name__ == '__main__': | |||||
sys.exit(_main()) |
#!/usr/bin/env python | |||||
from __future__ import absolute_import | |||||
import locale | |||||
import logging | |||||
import os | |||||
import optparse | |||||
import warnings | |||||
import sys | |||||
# 2016-06-17 barry@debian.org: urllib3 1.14 added optional support for socks, | |||||
# but if invoked (i.e. imported), it will issue a warning to stderr if socks | |||||
# isn't available. requests unconditionally imports urllib3's socks contrib | |||||
# module, triggering this warning. The warning breaks DEP-8 tests (because of | |||||
# the stderr output) and is just plain annoying in normal usage. I don't want | |||||
# to add socks as yet another dependency for pip, nor do I want to allow-stder | |||||
# in the DEP-8 tests, so just suppress the warning. pdb tells me this has to | |||||
# be done before the import of pip.vcs. | |||||
from pip._vendor.urllib3.exceptions import DependencyWarning | |||||
warnings.filterwarnings("ignore", category=DependencyWarning) # noqa | |||||
# We want to inject the use of SecureTransport as early as possible so that any | |||||
# references or sessions or what have you are ensured to have it, however we | |||||
# only want to do this in the case that we're running on macOS and the linked | |||||
# OpenSSL is too old to handle TLSv1.2 | |||||
try: | |||||
import ssl | |||||
except ImportError: | |||||
pass | |||||
else: | |||||
# Checks for OpenSSL 1.0.1 on MacOS | |||||
if sys.platform == "darwin" and ssl.OPENSSL_VERSION_NUMBER < 0x1000100f: | |||||
try: | |||||
from pip._vendor.urllib3.contrib import securetransport | |||||
except (ImportError, OSError): | |||||
pass | |||||
else: | |||||
securetransport.inject_into_urllib3() | |||||
from pip import __version__ | |||||
from pip._internal import cmdoptions | |||||
from pip._internal.exceptions import CommandError, PipError | |||||
from pip._internal.utils.misc import get_installed_distributions, get_prog | |||||
from pip._internal.utils import deprecation | |||||
from pip._internal.vcs import git, mercurial, subversion, bazaar # noqa | |||||
from pip._internal.baseparser import ( | |||||
ConfigOptionParser, UpdatingDefaultsHelpFormatter, | |||||
) | |||||
from pip._internal.commands import get_summaries, get_similar_commands | |||||
from pip._internal.commands import commands_dict | |||||
from pip._vendor.urllib3.exceptions import InsecureRequestWarning | |||||
logger = logging.getLogger(__name__) | |||||
# Hide the InsecureRequestWarning from urllib3 | |||||
warnings.filterwarnings("ignore", category=InsecureRequestWarning) | |||||
def autocomplete(): | |||||
"""Command and option completion for the main option parser (and options) | |||||
and its subcommands (and options). | |||||
Enable by sourcing one of the completion shell scripts (bash, zsh or fish). | |||||
""" | |||||
# Don't complete if user hasn't sourced bash_completion file. | |||||
if 'PIP_AUTO_COMPLETE' not in os.environ: | |||||
return | |||||
cwords = os.environ['COMP_WORDS'].split()[1:] | |||||
cword = int(os.environ['COMP_CWORD']) | |||||
try: | |||||
current = cwords[cword - 1] | |||||
except IndexError: | |||||
current = '' | |||||
subcommands = [cmd for cmd, summary in get_summaries()] | |||||
options = [] | |||||
# subcommand | |||||
try: | |||||
subcommand_name = [w for w in cwords if w in subcommands][0] | |||||
except IndexError: | |||||
subcommand_name = None | |||||
parser = create_main_parser() | |||||
# subcommand options | |||||
if subcommand_name: | |||||
# special case: 'help' subcommand has no options | |||||
if subcommand_name == 'help': | |||||
sys.exit(1) | |||||
# special case: list locally installed dists for show and uninstall | |||||
should_list_installed = ( | |||||
subcommand_name in ['show', 'uninstall'] and | |||||
not current.startswith('-') | |||||
) | |||||
if should_list_installed: | |||||
installed = [] | |||||
lc = current.lower() | |||||
for dist in get_installed_distributions(local_only=True): | |||||
if dist.key.startswith(lc) and dist.key not in cwords[1:]: | |||||
installed.append(dist.key) | |||||
# if there are no dists installed, fall back to option completion | |||||
if installed: | |||||
for dist in installed: | |||||
print(dist) | |||||
sys.exit(1) | |||||
subcommand = commands_dict[subcommand_name]() | |||||
for opt in subcommand.parser.option_list_all: | |||||
if opt.help != optparse.SUPPRESS_HELP: | |||||
for opt_str in opt._long_opts + opt._short_opts: | |||||
options.append((opt_str, opt.nargs)) | |||||
# filter out previously specified options from available options | |||||
prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] | |||||
options = [(x, v) for (x, v) in options if x not in prev_opts] | |||||
# filter options by current input | |||||
options = [(k, v) for k, v in options if k.startswith(current)] | |||||
for option in options: | |||||
opt_label = option[0] | |||||
# append '=' to options which require args | |||||
if option[1] and option[0][:2] == "--": | |||||
opt_label += '=' | |||||
print(opt_label) | |||||
else: | |||||
# show main parser options only when necessary | |||||
if current.startswith('-') or current.startswith('--'): | |||||
opts = [i.option_list for i in parser.option_groups] | |||||
opts.append(parser.option_list) | |||||
opts = (o for it in opts for o in it) | |||||
for opt in opts: | |||||
if opt.help != optparse.SUPPRESS_HELP: | |||||
subcommands += opt._long_opts + opt._short_opts | |||||
print(' '.join([x for x in subcommands if x.startswith(current)])) | |||||
sys.exit(1) | |||||
def create_main_parser(): | |||||
parser_kw = { | |||||
'usage': '\n%prog <command> [options]', | |||||
'add_help_option': False, | |||||
'formatter': UpdatingDefaultsHelpFormatter(), | |||||
'name': 'global', | |||||
'prog': get_prog(), | |||||
} | |||||
parser = ConfigOptionParser(**parser_kw) | |||||
parser.disable_interspersed_args() | |||||
pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |||||
parser.version = 'pip %s from %s (python %s)' % ( | |||||
__version__, pip_pkg_dir, sys.version[:3], | |||||
) | |||||
# add the general options | |||||
gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) | |||||
parser.add_option_group(gen_opts) | |||||
parser.main = True # so the help formatter knows | |||||
# create command listing for description | |||||
command_summaries = get_summaries() | |||||
description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] | |||||
parser.description = '\n'.join(description) | |||||
return parser | |||||
def parseopts(args): | |||||
parser = create_main_parser() | |||||
# Note: parser calls disable_interspersed_args(), so the result of this | |||||
# call is to split the initial args into the general options before the | |||||
# subcommand and everything else. | |||||
# For example: | |||||
# args: ['--timeout=5', 'install', '--user', 'INITools'] | |||||
# general_options: ['--timeout==5'] | |||||
# args_else: ['install', '--user', 'INITools'] | |||||
general_options, args_else = parser.parse_args(args) | |||||
# --version | |||||
if general_options.version: | |||||
sys.stdout.write(parser.version) | |||||
sys.stdout.write(os.linesep) | |||||
sys.exit() | |||||
# pip || pip help -> print_help() | |||||
if not args_else or (args_else[0] == 'help' and len(args_else) == 1): | |||||
parser.print_help() | |||||
sys.exit() | |||||
# the subcommand name | |||||
cmd_name = args_else[0] | |||||
if cmd_name not in commands_dict: | |||||
guess = get_similar_commands(cmd_name) | |||||
msg = ['unknown command "%s"' % cmd_name] | |||||
if guess: | |||||
msg.append('maybe you meant "%s"' % guess) | |||||
raise CommandError(' - '.join(msg)) | |||||
# all the args without the subcommand | |||||
cmd_args = args[:] | |||||
cmd_args.remove(cmd_name) | |||||
return cmd_name, cmd_args | |||||
def check_isolated(args): | |||||
isolated = False | |||||
if "--isolated" in args: | |||||
isolated = True | |||||
return isolated | |||||
def main(args=None): | |||||
if args is None: | |||||
args = sys.argv[1:] | |||||
# Configure our deprecation warnings to be sent through loggers | |||||
deprecation.install_warning_logger() | |||||
autocomplete() | |||||
try: | |||||
cmd_name, cmd_args = parseopts(args) | |||||
except PipError as exc: | |||||
sys.stderr.write("ERROR: %s" % exc) | |||||
sys.stderr.write(os.linesep) | |||||
sys.exit(1) | |||||
# Needed for locale.getpreferredencoding(False) to work | |||||
# in pip._internal.utils.encoding.auto_decode | |||||
try: | |||||
locale.setlocale(locale.LC_ALL, '') | |||||
except locale.Error as e: | |||||
# setlocale can apparently crash if locale are uninitialized | |||||
logger.debug("Ignoring error %s when setting locale", e) | |||||
command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) | |||||
return command.main(cmd_args) |
"""Base Command class, and related routines""" | |||||
from __future__ import absolute_import | |||||
import logging | |||||
import logging.config | |||||
import optparse | |||||
import os | |||||
import sys | |||||
import warnings | |||||
from pip._internal import cmdoptions | |||||
from pip._internal.baseparser import ( | |||||
ConfigOptionParser, UpdatingDefaultsHelpFormatter, | |||||
) | |||||
from pip._internal.compat import WINDOWS | |||||
from pip._internal.download import PipSession | |||||
from pip._internal.exceptions import ( | |||||
BadCommand, CommandError, InstallationError, PreviousBuildDirError, | |||||
UninstallationError, | |||||
) | |||||
from pip._internal.index import PackageFinder | |||||
from pip._internal.locations import running_under_virtualenv | |||||
from pip._internal.req.req_file import parse_requirements | |||||
from pip._internal.req.req_install import InstallRequirement | |||||
from pip._internal.status_codes import ( | |||||
ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, | |||||
VIRTUALENV_NOT_FOUND, | |||||
) | |||||
from pip._internal.utils import deprecation | |||||
from pip._internal.utils.logging import IndentingFormatter | |||||
from pip._internal.utils.misc import get_prog, normalize_path | |||||
from pip._internal.utils.outdated import pip_version_check | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Optional | |||||
__all__ = ['Command'] | |||||
logger = logging.getLogger(__name__) | |||||
class Command(object): | |||||
name = None # type: Optional[str] | |||||
usage = None # type: Optional[str] | |||||
hidden = False # type: bool | |||||
ignore_require_venv = False # type: bool | |||||
log_streams = ("ext://sys.stdout", "ext://sys.stderr") | |||||
def __init__(self, isolated=False): | |||||
parser_kw = { | |||||
'usage': self.usage, | |||||
'prog': '%s %s' % (get_prog(), self.name), | |||||
'formatter': UpdatingDefaultsHelpFormatter(), | |||||
'add_help_option': False, | |||||
'name': self.name, | |||||
'description': self.__doc__, | |||||
'isolated': isolated, | |||||
} | |||||
self.parser = ConfigOptionParser(**parser_kw) | |||||
# Commands should add options to this option group | |||||
optgroup_name = '%s Options' % self.name.capitalize() | |||||
self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name) | |||||
# Add the general options | |||||
gen_opts = cmdoptions.make_option_group( | |||||
cmdoptions.general_group, | |||||
self.parser, | |||||
) | |||||
self.parser.add_option_group(gen_opts) | |||||
def _build_session(self, options, retries=None, timeout=None): | |||||
session = PipSession( | |||||
cache=( | |||||
normalize_path(os.path.join(options.cache_dir, "http")) | |||||
if options.cache_dir else None | |||||
), | |||||
retries=retries if retries is not None else options.retries, | |||||
insecure_hosts=options.trusted_hosts, | |||||
) | |||||
# Handle custom ca-bundles from the user | |||||
if options.cert: | |||||
session.verify = options.cert | |||||
# Handle SSL client certificate | |||||
if options.client_cert: | |||||
session.cert = options.client_cert | |||||
# Handle timeouts | |||||
if options.timeout or timeout: | |||||
session.timeout = ( | |||||
timeout if timeout is not None else options.timeout | |||||
) | |||||
# Handle configured proxies | |||||
if options.proxy: | |||||
session.proxies = { | |||||
"http": options.proxy, | |||||
"https": options.proxy, | |||||
} | |||||
# Determine if we can prompt the user for authentication or not | |||||
session.auth.prompting = not options.no_input | |||||
return session | |||||
def parse_args(self, args): | |||||
# factored out for testability | |||||
return self.parser.parse_args(args) | |||||
def main(self, args): | |||||
options, args = self.parse_args(args) | |||||
# Set verbosity so that it can be used elsewhere. | |||||
self.verbosity = options.verbose - options.quiet | |||||
if self.verbosity >= 1: | |||||
level = "DEBUG" | |||||
elif self.verbosity == -1: | |||||
level = "WARNING" | |||||
elif self.verbosity == -2: | |||||
level = "ERROR" | |||||
elif self.verbosity <= -3: | |||||
level = "CRITICAL" | |||||
else: | |||||
level = "INFO" | |||||
# The root logger should match the "console" level *unless* we | |||||
# specified "--log" to send debug logs to a file. | |||||
root_level = level | |||||
if options.log: | |||||
root_level = "DEBUG" | |||||
logger_class = "pip._internal.utils.logging.ColorizedStreamHandler" | |||||
handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler" | |||||
logging.config.dictConfig({ | |||||
"version": 1, | |||||
"disable_existing_loggers": False, | |||||
"filters": { | |||||
"exclude_warnings": { | |||||
"()": "pip._internal.utils.logging.MaxLevelFilter", | |||||
"level": logging.WARNING, | |||||
}, | |||||
}, | |||||
"formatters": { | |||||
"indent": { | |||||
"()": IndentingFormatter, | |||||
"format": "%(message)s", | |||||
}, | |||||
}, | |||||
"handlers": { | |||||
"console": { | |||||
"level": level, | |||||
"class": logger_class, | |||||
"no_color": options.no_color, | |||||
"stream": self.log_streams[0], | |||||
"filters": ["exclude_warnings"], | |||||
"formatter": "indent", | |||||
}, | |||||
"console_errors": { | |||||
"level": "WARNING", | |||||
"class": logger_class, | |||||
"no_color": options.no_color, | |||||
"stream": self.log_streams[1], | |||||
"formatter": "indent", | |||||
}, | |||||
"user_log": { | |||||
"level": "DEBUG", | |||||
"class": handler_class, | |||||
"filename": options.log or "/dev/null", | |||||
"delay": True, | |||||
"formatter": "indent", | |||||
}, | |||||
}, | |||||
"root": { | |||||
"level": root_level, | |||||
"handlers": list(filter(None, [ | |||||
"console", | |||||
"console_errors", | |||||
"user_log" if options.log else None, | |||||
])), | |||||
}, | |||||
# Disable any logging besides WARNING unless we have DEBUG level | |||||
# logging enabled. These use both pip._vendor and the bare names | |||||
# for the case where someone unbundles our libraries. | |||||
"loggers": { | |||||
name: { | |||||
"level": ( | |||||
"WARNING" if level in ["INFO", "ERROR"] else "DEBUG" | |||||
) | |||||
} for name in [ | |||||
"pip._vendor", "distlib", "requests", "urllib3" | |||||
] | |||||
}, | |||||
}) | |||||
if sys.version_info[:2] == (3, 3): | |||||
warnings.warn( | |||||
"Python 3.3 supported has been deprecated and support for it " | |||||
"will be dropped in the future. Please upgrade your Python.", | |||||
deprecation.RemovedInPip11Warning, | |||||
) | |||||
# TODO: try to get these passing down from the command? | |||||
# without resorting to os.environ to hold these. | |||||
if options.no_input: | |||||
os.environ['PIP_NO_INPUT'] = '1' | |||||
if options.exists_action: | |||||
os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) | |||||
if options.require_venv and not self.ignore_require_venv: | |||||
# If a venv is required check if it can really be found | |||||
if not running_under_virtualenv(): | |||||
logger.critical( | |||||
'Could not find an activated virtualenv (required).' | |||||
) | |||||
sys.exit(VIRTUALENV_NOT_FOUND) | |||||
original_root_handlers = set(logging.root.handlers) | |||||
try: | |||||
status = self.run(options, args) | |||||
# FIXME: all commands should return an exit status | |||||
# and when it is done, isinstance is not needed anymore | |||||
if isinstance(status, int): | |||||
return status | |||||
except PreviousBuildDirError as exc: | |||||
logger.critical(str(exc)) | |||||
logger.debug('Exception information:', exc_info=True) | |||||
return PREVIOUS_BUILD_DIR_ERROR | |||||
except (InstallationError, UninstallationError, BadCommand) as exc: | |||||
logger.critical(str(exc)) | |||||
logger.debug('Exception information:', exc_info=True) | |||||
return ERROR | |||||
except CommandError as exc: | |||||
logger.critical('ERROR: %s', exc) | |||||
logger.debug('Exception information:', exc_info=True) | |||||
return ERROR | |||||
except KeyboardInterrupt: | |||||
logger.critical('Operation cancelled by user') | |||||
logger.debug('Exception information:', exc_info=True) | |||||
return ERROR | |||||
except: | |||||
logger.critical('Exception:', exc_info=True) | |||||
return UNKNOWN_ERROR | |||||
finally: | |||||
# Check if we're using the latest version of pip available | |||||
if (not options.disable_pip_version_check and not | |||||
getattr(options, "no_index", False)): | |||||
with self._build_session( | |||||
options, | |||||
retries=0, | |||||
timeout=min(5, options.timeout)) as session: | |||||
pip_version_check(session, options) | |||||
# Avoid leaking loggers | |||||
for handler in set(logging.root.handlers) - original_root_handlers: | |||||
# this method benefit from the Logger class internal lock | |||||
logging.root.removeHandler(handler) | |||||
return SUCCESS | |||||
class RequirementCommand(Command): | |||||
@staticmethod | |||||
def populate_requirement_set(requirement_set, args, options, finder, | |||||
session, name, wheel_cache): | |||||
""" | |||||
Marshal cmd line args into a requirement set. | |||||
""" | |||||
# NOTE: As a side-effect, options.require_hashes and | |||||
# requirement_set.require_hashes may be updated | |||||
for filename in options.constraints: | |||||
for req_to_add in parse_requirements( | |||||
filename, | |||||
constraint=True, finder=finder, options=options, | |||||
session=session, wheel_cache=wheel_cache): | |||||
req_to_add.is_direct = True | |||||
requirement_set.add_requirement(req_to_add) | |||||
for req in args: | |||||
req_to_add = InstallRequirement.from_line( | |||||
req, None, isolated=options.isolated_mode, | |||||
wheel_cache=wheel_cache | |||||
) | |||||
req_to_add.is_direct = True | |||||
requirement_set.add_requirement(req_to_add) | |||||
for req in options.editables: | |||||
req_to_add = InstallRequirement.from_editable( | |||||
req, | |||||
isolated=options.isolated_mode, | |||||
wheel_cache=wheel_cache | |||||
) | |||||
req_to_add.is_direct = True | |||||
requirement_set.add_requirement(req_to_add) | |||||
for filename in options.requirements: | |||||
for req_to_add in parse_requirements( | |||||
filename, | |||||
finder=finder, options=options, session=session, | |||||
wheel_cache=wheel_cache): | |||||
req_to_add.is_direct = True | |||||
requirement_set.add_requirement(req_to_add) | |||||
# If --require-hashes was a line in a requirements file, tell | |||||
# RequirementSet about it: | |||||
requirement_set.require_hashes = options.require_hashes | |||||
if not (args or options.editables or options.requirements): | |||||
opts = {'name': name} | |||||
if options.find_links: | |||||
raise CommandError( | |||||
'You must give at least one requirement to %(name)s ' | |||||
'(maybe you meant "pip %(name)s %(links)s"?)' % | |||||
dict(opts, links=' '.join(options.find_links))) | |||||
else: | |||||
raise CommandError( | |||||
'You must give at least one requirement to %(name)s ' | |||||
'(see "pip help %(name)s")' % opts) | |||||
# On Windows, any operation modifying pip should be run as: | |||||
# python -m pip ... | |||||
# See https://github.com/pypa/pip/issues/1299 for more discussion | |||||
should_show_use_python_msg = ( | |||||
WINDOWS and | |||||
requirement_set.has_requirement("pip") and | |||||
os.path.basename(sys.argv[0]).startswith("pip") | |||||
) | |||||
if should_show_use_python_msg: | |||||
new_command = [ | |||||
sys.executable, "-m", "pip" | |||||
] + sys.argv[1:] | |||||
raise CommandError( | |||||
'To modify pip, please run the following command:\n{}' | |||||
.format(" ".join(new_command)) | |||||
) | |||||
def _build_package_finder(self, options, session, | |||||
platform=None, python_versions=None, | |||||
abi=None, implementation=None): | |||||
""" | |||||
Create a package finder appropriate to this requirement command. | |||||
""" | |||||
index_urls = [options.index_url] + options.extra_index_urls | |||||
if options.no_index: | |||||
logger.debug('Ignoring indexes: %s', ','.join(index_urls)) | |||||
index_urls = [] | |||||
return PackageFinder( | |||||
find_links=options.find_links, | |||||
format_control=options.format_control, | |||||
index_urls=index_urls, | |||||
trusted_hosts=options.trusted_hosts, | |||||
allow_all_prereleases=options.pre, | |||||
process_dependency_links=options.process_dependency_links, | |||||
session=session, | |||||
platform=platform, | |||||
versions=python_versions, | |||||
abi=abi, | |||||
implementation=implementation, | |||||
) |
"""Base option parser setup""" | |||||
from __future__ import absolute_import | |||||
import logging | |||||
import optparse | |||||
import sys | |||||
import textwrap | |||||
from distutils.util import strtobool | |||||
from pip._vendor.six import string_types | |||||
from pip._internal.compat import get_terminal_size | |||||
from pip._internal.configuration import Configuration, ConfigurationError | |||||
logger = logging.getLogger(__name__) | |||||
class PrettyHelpFormatter(optparse.IndentedHelpFormatter): | |||||
"""A prettier/less verbose help formatter for optparse.""" | |||||
def __init__(self, *args, **kwargs): | |||||
# help position must be aligned with __init__.parseopts.description | |||||
kwargs['max_help_position'] = 30 | |||||
kwargs['indent_increment'] = 1 | |||||
kwargs['width'] = get_terminal_size()[0] - 2 | |||||
optparse.IndentedHelpFormatter.__init__(self, *args, **kwargs) | |||||
def format_option_strings(self, option): | |||||
return self._format_option_strings(option, ' <%s>', ', ') | |||||
def _format_option_strings(self, option, mvarfmt=' <%s>', optsep=', '): | |||||
""" | |||||
Return a comma-separated list of option strings and metavars. | |||||
:param option: tuple of (short opt, long opt), e.g: ('-f', '--format') | |||||
:param mvarfmt: metavar format string - evaluated as mvarfmt % metavar | |||||
:param optsep: separator | |||||
""" | |||||
opts = [] | |||||
if option._short_opts: | |||||
opts.append(option._short_opts[0]) | |||||
if option._long_opts: | |||||
opts.append(option._long_opts[0]) | |||||
if len(opts) > 1: | |||||
opts.insert(1, optsep) | |||||
if option.takes_value(): | |||||
metavar = option.metavar or option.dest.lower() | |||||
opts.append(mvarfmt % metavar.lower()) | |||||
return ''.join(opts) | |||||
def format_heading(self, heading): | |||||
if heading == 'Options': | |||||
return '' | |||||
return heading + ':\n' | |||||
def format_usage(self, usage): | |||||
""" | |||||
Ensure there is only one newline between usage and the first heading | |||||
if there is no description. | |||||
""" | |||||
msg = '\nUsage: %s\n' % self.indent_lines(textwrap.dedent(usage), " ") | |||||
return msg | |||||
def format_description(self, description): | |||||
# leave full control over description to us | |||||
if description: | |||||
if hasattr(self.parser, 'main'): | |||||
label = 'Commands' | |||||
else: | |||||
label = 'Description' | |||||
# some doc strings have initial newlines, some don't | |||||
description = description.lstrip('\n') | |||||
# some doc strings have final newlines and spaces, some don't | |||||
description = description.rstrip() | |||||
# dedent, then reindent | |||||
description = self.indent_lines(textwrap.dedent(description), " ") | |||||
description = '%s:\n%s\n' % (label, description) | |||||
return description | |||||
else: | |||||
return '' | |||||
def format_epilog(self, epilog): | |||||
# leave full control over epilog to us | |||||
if epilog: | |||||
return epilog | |||||
else: | |||||
return '' | |||||
def indent_lines(self, text, indent): | |||||
new_lines = [indent + line for line in text.split('\n')] | |||||
return "\n".join(new_lines) | |||||
class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): | |||||
"""Custom help formatter for use in ConfigOptionParser. | |||||
This is updates the defaults before expanding them, allowing | |||||
them to show up correctly in the help listing. | |||||
""" | |||||
def expand_default(self, option): | |||||
if self.parser is not None: | |||||
self.parser._update_defaults(self.parser.defaults) | |||||
return optparse.IndentedHelpFormatter.expand_default(self, option) | |||||
class CustomOptionParser(optparse.OptionParser): | |||||
def insert_option_group(self, idx, *args, **kwargs): | |||||
"""Insert an OptionGroup at a given position.""" | |||||
group = self.add_option_group(*args, **kwargs) | |||||
self.option_groups.pop() | |||||
self.option_groups.insert(idx, group) | |||||
return group | |||||
@property | |||||
def option_list_all(self): | |||||
"""Get a list of all options, including those in option groups.""" | |||||
res = self.option_list[:] | |||||
for i in self.option_groups: | |||||
res.extend(i.option_list) | |||||
return res | |||||
class ConfigOptionParser(CustomOptionParser): | |||||
"""Custom option parser which updates its defaults by checking the | |||||
configuration files and environmental variables""" | |||||
def __init__(self, *args, **kwargs): | |||||
self.name = kwargs.pop('name') | |||||
isolated = kwargs.pop("isolated", False) | |||||
self.config = Configuration(isolated) | |||||
assert self.name | |||||
optparse.OptionParser.__init__(self, *args, **kwargs) | |||||
def check_default(self, option, key, val): | |||||
try: | |||||
return option.check_value(key, val) | |||||
except optparse.OptionValueError as exc: | |||||
print("An error occurred during configuration: %s" % exc) | |||||
sys.exit(3) | |||||
def _get_ordered_configuration_items(self): | |||||
# Configuration gives keys in an unordered manner. Order them. | |||||
override_order = ["global", self.name, ":env:"] | |||||
# Pool the options into different groups | |||||
section_items = {name: [] for name in override_order} | |||||
for section_key, val in self.config.items(): | |||||
# ignore empty values | |||||
if not val: | |||||
logger.debug( | |||||
"Ignoring configuration key '%s' as it's value is empty.", | |||||
section_key | |||||
) | |||||
continue | |||||
section, key = section_key.split(".", 1) | |||||
if section in override_order: | |||||
section_items[section].append((key, val)) | |||||
# Yield each group in their override order | |||||
for section in override_order: | |||||
for key, val in section_items[section]: | |||||
yield key, val | |||||
def _update_defaults(self, defaults): | |||||
"""Updates the given defaults with values from the config files and | |||||
the environ. Does a little special handling for certain types of | |||||
options (lists).""" | |||||
# Accumulate complex default state. | |||||
self.values = optparse.Values(self.defaults) | |||||
late_eval = set() | |||||
# Then set the options with those values | |||||
for key, val in self._get_ordered_configuration_items(): | |||||
# '--' because configuration supports only long names | |||||
option = self.get_option('--' + key) | |||||
# Ignore options not present in this parser. E.g. non-globals put | |||||
# in [global] by users that want them to apply to all applicable | |||||
# commands. | |||||
if option is None: | |||||
continue | |||||
if option.action in ('store_true', 'store_false', 'count'): | |||||
val = strtobool(val) | |||||
elif option.action == 'append': | |||||
val = val.split() | |||||
val = [self.check_default(option, key, v) for v in val] | |||||
elif option.action == 'callback': | |||||
late_eval.add(option.dest) | |||||
opt_str = option.get_opt_string() | |||||
val = option.convert_value(opt_str, val) | |||||
# From take_action | |||||
args = option.callback_args or () | |||||
kwargs = option.callback_kwargs or {} | |||||
option.callback(option, opt_str, val, self, *args, **kwargs) | |||||
else: | |||||
val = self.check_default(option, key, val) | |||||
defaults[option.dest] = val | |||||
for key in late_eval: | |||||
defaults[key] = getattr(self.values, key) | |||||
self.values = None | |||||
return defaults | |||||
def get_default_values(self): | |||||
"""Overriding to make updating the defaults after instantiation of | |||||
the option parser possible, _update_defaults() does the dirty work.""" | |||||
if not self.process_default_values: | |||||
# Old, pre-Optik 1.5 behaviour. | |||||
return optparse.Values(self.defaults) | |||||
# Load the configuration, or error out in case of an error | |||||
try: | |||||
self.config.load() | |||||
except ConfigurationError as err: | |||||
self.exit(2, err.args[0]) | |||||
defaults = self._update_defaults(self.defaults.copy()) # ours | |||||
for option in self._get_all_options(): | |||||
default = defaults.get(option.dest) | |||||
if isinstance(default, string_types): | |||||
opt_str = option.get_opt_string() | |||||
defaults[option.dest] = option.check_value(opt_str, default) | |||||
return optparse.Values(defaults) | |||||
def error(self, msg): | |||||
self.print_usage(sys.stderr) | |||||
self.exit(2, "%s\n" % msg) |
"""Build Environment used for isolation during sdist building | |||||
""" | |||||
import os | |||||
from distutils.sysconfig import get_python_lib | |||||
from sysconfig import get_paths | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
class BuildEnvironment(object): | |||||
"""Creates and manages an isolated environment to install build deps | |||||
""" | |||||
def __init__(self, no_clean): | |||||
self._temp_dir = TempDirectory(kind="build-env") | |||||
self._no_clean = no_clean | |||||
@property | |||||
def path(self): | |||||
return self._temp_dir.path | |||||
def __enter__(self): | |||||
self._temp_dir.create() | |||||
self.save_path = os.environ.get('PATH', None) | |||||
self.save_pythonpath = os.environ.get('PYTHONPATH', None) | |||||
self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) | |||||
install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' | |||||
install_dirs = get_paths(install_scheme, vars={ | |||||
'base': self.path, | |||||
'platbase': self.path, | |||||
}) | |||||
scripts = install_dirs['scripts'] | |||||
if self.save_path: | |||||
os.environ['PATH'] = scripts + os.pathsep + self.save_path | |||||
else: | |||||
os.environ['PATH'] = scripts + os.pathsep + os.defpath | |||||
# Note: prefer distutils' sysconfig to get the | |||||
# library paths so PyPy is correctly supported. | |||||
purelib = get_python_lib(plat_specific=0, prefix=self.path) | |||||
platlib = get_python_lib(plat_specific=1, prefix=self.path) | |||||
if purelib == platlib: | |||||
lib_dirs = purelib | |||||
else: | |||||
lib_dirs = purelib + os.pathsep + platlib | |||||
if self.save_pythonpath: | |||||
os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ | |||||
self.save_pythonpath | |||||
else: | |||||
os.environ['PYTHONPATH'] = lib_dirs | |||||
os.environ['PYTHONNOUSERSITE'] = '1' | |||||
return self.path | |||||
def __exit__(self, exc_type, exc_val, exc_tb): | |||||
if not self._no_clean: | |||||
self._temp_dir.cleanup() | |||||
def restore_var(varname, old_value): | |||||
if old_value is None: | |||||
os.environ.pop(varname, None) | |||||
else: | |||||
os.environ[varname] = old_value | |||||
restore_var('PATH', self.save_path) | |||||
restore_var('PYTHONPATH', self.save_pythonpath) | |||||
restore_var('PYTHONNOUSERSITE', self.save_nousersite) | |||||
def cleanup(self): | |||||
self._temp_dir.cleanup() | |||||
class NoOpBuildEnvironment(BuildEnvironment): | |||||
"""A no-op drop-in replacement for BuildEnvironment | |||||
""" | |||||
def __init__(self, no_clean): | |||||
pass | |||||
def __enter__(self): | |||||
pass | |||||
def __exit__(self, exc_type, exc_val, exc_tb): | |||||
pass | |||||
def cleanup(self): | |||||
pass |
"""Cache Management | |||||
""" | |||||
import errno | |||||
import hashlib | |||||
import logging | |||||
import os | |||||
from pip._vendor.packaging.utils import canonicalize_name | |||||
from pip._internal import index | |||||
from pip._internal.compat import expanduser | |||||
from pip._internal.download import path_to_url | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.wheel import InvalidWheelFilename, Wheel | |||||
logger = logging.getLogger(__name__) | |||||
class Cache(object): | |||||
"""An abstract class - provides cache directories for data from links | |||||
:param cache_dir: The root of the cache. | |||||
:param format_control: A pip.index.FormatControl object to limit | |||||
binaries being read from the cache. | |||||
:param allowed_formats: which formats of files the cache should store. | |||||
('binary' and 'source' are the only allowed values) | |||||
""" | |||||
def __init__(self, cache_dir, format_control, allowed_formats): | |||||
super(Cache, self).__init__() | |||||
self.cache_dir = expanduser(cache_dir) if cache_dir else None | |||||
self.format_control = format_control | |||||
self.allowed_formats = allowed_formats | |||||
_valid_formats = {"source", "binary"} | |||||
assert self.allowed_formats.union(_valid_formats) == _valid_formats | |||||
def _get_cache_path_parts(self, link): | |||||
"""Get parts of part that must be os.path.joined with cache_dir | |||||
""" | |||||
# We want to generate an url to use as our cache key, we don't want to | |||||
# just re-use the URL because it might have other items in the fragment | |||||
# and we don't care about those. | |||||
key_parts = [link.url_without_fragment] | |||||
if link.hash_name is not None and link.hash is not None: | |||||
key_parts.append("=".join([link.hash_name, link.hash])) | |||||
key_url = "#".join(key_parts) | |||||
# Encode our key url with sha224, we'll use this because it has similar | |||||
# security properties to sha256, but with a shorter total output (and | |||||
# thus less secure). However the differences don't make a lot of | |||||
# difference for our use case here. | |||||
hashed = hashlib.sha224(key_url.encode()).hexdigest() | |||||
# We want to nest the directories some to prevent having a ton of top | |||||
# level directories where we might run out of sub directories on some | |||||
# FS. | |||||
parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]] | |||||
return parts | |||||
def _get_candidates(self, link, package_name): | |||||
can_not_cache = ( | |||||
not self.cache_dir or | |||||
not package_name or | |||||
not link | |||||
) | |||||
if can_not_cache: | |||||
return [] | |||||
canonical_name = canonicalize_name(package_name) | |||||
formats = index.fmt_ctl_formats( | |||||
self.format_control, canonical_name | |||||
) | |||||
if not self.allowed_formats.intersection(formats): | |||||
return [] | |||||
root = self.get_path_for_link(link) | |||||
try: | |||||
return os.listdir(root) | |||||
except OSError as err: | |||||
if err.errno in {errno.ENOENT, errno.ENOTDIR}: | |||||
return [] | |||||
raise | |||||
def get_path_for_link(self, link): | |||||
"""Return a directory to store cached items in for link. | |||||
""" | |||||
raise NotImplementedError() | |||||
def get(self, link, package_name): | |||||
"""Returns a link to a cached item if it exists, otherwise returns the | |||||
passed link. | |||||
""" | |||||
raise NotImplementedError() | |||||
def _link_for_candidate(self, link, candidate): | |||||
root = self.get_path_for_link(link) | |||||
path = os.path.join(root, candidate) | |||||
return index.Link(path_to_url(path)) | |||||
def cleanup(self): | |||||
pass | |||||
class SimpleWheelCache(Cache): | |||||
"""A cache of wheels for future installs. | |||||
""" | |||||
def __init__(self, cache_dir, format_control): | |||||
super(SimpleWheelCache, self).__init__( | |||||
cache_dir, format_control, {"binary"} | |||||
) | |||||
def get_path_for_link(self, link): | |||||
"""Return a directory to store cached wheels for link | |||||
Because there are M wheels for any one sdist, we provide a directory | |||||
to cache them in, and then consult that directory when looking up | |||||
cache hits. | |||||
We only insert things into the cache if they have plausible version | |||||
numbers, so that we don't contaminate the cache with things that were | |||||
not unique. E.g. ./package might have dozens of installs done for it | |||||
and build a version of 0.0...and if we built and cached a wheel, we'd | |||||
end up using the same wheel even if the source has been edited. | |||||
:param link: The link of the sdist for which this will cache wheels. | |||||
""" | |||||
parts = self._get_cache_path_parts(link) | |||||
# Store wheels within the root cache_dir | |||||
return os.path.join(self.cache_dir, "wheels", *parts) | |||||
def get(self, link, package_name): | |||||
candidates = [] | |||||
for wheel_name in self._get_candidates(link, package_name): | |||||
try: | |||||
wheel = Wheel(wheel_name) | |||||
except InvalidWheelFilename: | |||||
continue | |||||
if not wheel.supported(): | |||||
# Built for a different python/arch/etc | |||||
continue | |||||
candidates.append((wheel.support_index_min(), wheel_name)) | |||||
if not candidates: | |||||
return link | |||||
return self._link_for_candidate(link, min(candidates)[1]) | |||||
class EphemWheelCache(SimpleWheelCache): | |||||
"""A SimpleWheelCache that creates it's own temporary cache directory | |||||
""" | |||||
def __init__(self, format_control): | |||||
self._temp_dir = TempDirectory(kind="ephem-wheel-cache") | |||||
self._temp_dir.create() | |||||
super(EphemWheelCache, self).__init__( | |||||
self._temp_dir.path, format_control | |||||
) | |||||
def cleanup(self): | |||||
self._temp_dir.cleanup() | |||||
class WheelCache(Cache): | |||||
"""Wraps EphemWheelCache and SimpleWheelCache into a single Cache | |||||
This Cache allows for gracefully degradation, using the ephem wheel cache | |||||
when a certain link is not found in the simple wheel cache first. | |||||
""" | |||||
def __init__(self, cache_dir, format_control): | |||||
super(WheelCache, self).__init__( | |||||
cache_dir, format_control, {'binary'} | |||||
) | |||||
self._wheel_cache = SimpleWheelCache(cache_dir, format_control) | |||||
self._ephem_cache = EphemWheelCache(format_control) | |||||
def get_path_for_link(self, link): | |||||
return self._wheel_cache.get_path_for_link(link) | |||||
def get_ephem_path_for_link(self, link): | |||||
return self._ephem_cache.get_path_for_link(link) | |||||
def get(self, link, package_name): | |||||
retval = self._wheel_cache.get(link, package_name) | |||||
if retval is link: | |||||
retval = self._ephem_cache.get(link, package_name) | |||||
return retval | |||||
def cleanup(self): | |||||
self._wheel_cache.cleanup() | |||||
self._ephem_cache.cleanup() |
""" | |||||
shared options and groups | |||||
The principle here is to define options once, but *not* instantiate them | |||||
globally. One reason being that options with action='append' can carry state | |||||
between parses. pip parses general options twice internally, and shouldn't | |||||
pass on state. To be consistent, all options will follow this design. | |||||
""" | |||||
from __future__ import absolute_import | |||||
import warnings | |||||
from functools import partial | |||||
from optparse import SUPPRESS_HELP, Option, OptionGroup | |||||
from pip._internal.index import ( | |||||
FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary, | |||||
) | |||||
from pip._internal.locations import USER_CACHE_DIR, src_prefix | |||||
from pip._internal.models import PyPI | |||||
from pip._internal.utils.hashes import STRONG_HASHES | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
from pip._internal.utils.ui import BAR_TYPES | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Any | |||||
def make_option_group(group, parser): | |||||
""" | |||||
Return an OptionGroup object | |||||
group -- assumed to be dict with 'name' and 'options' keys | |||||
parser -- an optparse Parser | |||||
""" | |||||
option_group = OptionGroup(parser, group['name']) | |||||
for option in group['options']: | |||||
option_group.add_option(option()) | |||||
return option_group | |||||
def check_install_build_global(options, check_options=None): | |||||
"""Disable wheels if per-setup.py call options are set. | |||||
:param options: The OptionParser options to update. | |||||
:param check_options: The options to check, if not supplied defaults to | |||||
options. | |||||
""" | |||||
if check_options is None: | |||||
check_options = options | |||||
def getname(n): | |||||
return getattr(check_options, n, None) | |||||
names = ["build_options", "global_options", "install_options"] | |||||
if any(map(getname, names)): | |||||
control = options.format_control | |||||
fmt_ctl_no_binary(control) | |||||
warnings.warn( | |||||
'Disabling all use of wheels due to the use of --build-options ' | |||||
'/ --global-options / --install-options.', stacklevel=2, | |||||
) | |||||
########### | |||||
# options # | |||||
########### | |||||
help_ = partial( | |||||
Option, | |||||
'-h', '--help', | |||||
dest='help', | |||||
action='help', | |||||
help='Show help.', | |||||
) # type: Any | |||||
isolated_mode = partial( | |||||
Option, | |||||
"--isolated", | |||||
dest="isolated_mode", | |||||
action="store_true", | |||||
default=False, | |||||
help=( | |||||
"Run pip in an isolated mode, ignoring environment variables and user " | |||||
"configuration." | |||||
), | |||||
) | |||||
require_virtualenv = partial( | |||||
Option, | |||||
# Run only if inside a virtualenv, bail if not. | |||||
'--require-virtualenv', '--require-venv', | |||||
dest='require_venv', | |||||
action='store_true', | |||||
default=False, | |||||
help=SUPPRESS_HELP | |||||
) # type: Any | |||||
verbose = partial( | |||||
Option, | |||||
'-v', '--verbose', | |||||
dest='verbose', | |||||
action='count', | |||||
default=0, | |||||
help='Give more output. Option is additive, and can be used up to 3 times.' | |||||
) | |||||
no_color = partial( | |||||
Option, | |||||
'--no-color', | |||||
dest='no_color', | |||||
action='store_true', | |||||
default=False, | |||||
help="Suppress colored output", | |||||
) | |||||
version = partial( | |||||
Option, | |||||
'-V', '--version', | |||||
dest='version', | |||||
action='store_true', | |||||
help='Show version and exit.', | |||||
) # type: Any | |||||
quiet = partial( | |||||
Option, | |||||
'-q', '--quiet', | |||||
dest='quiet', | |||||
action='count', | |||||
default=0, | |||||
help=( | |||||
'Give less output. Option is additive, and can be used up to 3' | |||||
' times (corresponding to WARNING, ERROR, and CRITICAL logging' | |||||
' levels).' | |||||
), | |||||
) # type: Any | |||||
progress_bar = partial( | |||||
Option, | |||||
'--progress-bar', | |||||
dest='progress_bar', | |||||
type='choice', | |||||
choices=list(BAR_TYPES.keys()), | |||||
default='on', | |||||
help=( | |||||
'Specify type of progress to be displayed [' + | |||||
'|'.join(BAR_TYPES.keys()) + '] (default: %default)' | |||||
), | |||||
) # type: Any | |||||
log = partial( | |||||
Option, | |||||
"--log", "--log-file", "--local-log", | |||||
dest="log", | |||||
metavar="path", | |||||
help="Path to a verbose appending log." | |||||
) # type: Any | |||||
no_input = partial( | |||||
Option, | |||||
# Don't ask for input | |||||
'--no-input', | |||||
dest='no_input', | |||||
action='store_true', | |||||
default=False, | |||||
help=SUPPRESS_HELP | |||||
) # type: Any | |||||
proxy = partial( | |||||
Option, | |||||
'--proxy', | |||||
dest='proxy', | |||||
type='str', | |||||
default='', | |||||
help="Specify a proxy in the form [user:passwd@]proxy.server:port." | |||||
) # type: Any | |||||
retries = partial( | |||||
Option, | |||||
'--retries', | |||||
dest='retries', | |||||
type='int', | |||||
default=5, | |||||
help="Maximum number of retries each connection should attempt " | |||||
"(default %default times).", | |||||
) # type: Any | |||||
timeout = partial( | |||||
Option, | |||||
'--timeout', '--default-timeout', | |||||
metavar='sec', | |||||
dest='timeout', | |||||
type='float', | |||||
default=15, | |||||
help='Set the socket timeout (default %default seconds).', | |||||
) # type: Any | |||||
skip_requirements_regex = partial( | |||||
Option, | |||||
# A regex to be used to skip requirements | |||||
'--skip-requirements-regex', | |||||
dest='skip_requirements_regex', | |||||
type='str', | |||||
default='', | |||||
help=SUPPRESS_HELP, | |||||
) # type: Any | |||||
def exists_action(): | |||||
return Option( | |||||
# Option when path already exist | |||||
'--exists-action', | |||||
dest='exists_action', | |||||
type='choice', | |||||
choices=['s', 'i', 'w', 'b', 'a'], | |||||
default=[], | |||||
action='append', | |||||
metavar='action', | |||||
help="Default action when a path already exists: " | |||||
"(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort).", | |||||
) | |||||
cert = partial( | |||||
Option, | |||||
'--cert', | |||||
dest='cert', | |||||
type='str', | |||||
metavar='path', | |||||
help="Path to alternate CA bundle.", | |||||
) # type: Any | |||||
client_cert = partial( | |||||
Option, | |||||
'--client-cert', | |||||
dest='client_cert', | |||||
type='str', | |||||
default=None, | |||||
metavar='path', | |||||
help="Path to SSL client certificate, a single file containing the " | |||||
"private key and the certificate in PEM format.", | |||||
) # type: Any | |||||
index_url = partial( | |||||
Option, | |||||
'-i', '--index-url', '--pypi-url', | |||||
dest='index_url', | |||||
metavar='URL', | |||||
default=PyPI.simple_url, | |||||
help="Base URL of Python Package Index (default %default). " | |||||
"This should point to a repository compliant with PEP 503 " | |||||
"(the simple repository API) or a local directory laid out " | |||||
"in the same format.", | |||||
) # type: Any | |||||
def extra_index_url(): | |||||
return Option( | |||||
'--extra-index-url', | |||||
dest='extra_index_urls', | |||||
metavar='URL', | |||||
action='append', | |||||
default=[], | |||||
help="Extra URLs of package indexes to use in addition to " | |||||
"--index-url. Should follow the same rules as " | |||||
"--index-url.", | |||||
) | |||||
no_index = partial( | |||||
Option, | |||||
'--no-index', | |||||
dest='no_index', | |||||
action='store_true', | |||||
default=False, | |||||
help='Ignore package index (only looking at --find-links URLs instead).', | |||||
) # type: Any | |||||
def find_links(): | |||||
return Option( | |||||
'-f', '--find-links', | |||||
dest='find_links', | |||||
action='append', | |||||
default=[], | |||||
metavar='url', | |||||
help="If a url or path to an html file, then parse for links to " | |||||
"archives. If a local path or file:// url that's a directory, " | |||||
"then look for archives in the directory listing.", | |||||
) | |||||
def trusted_host(): | |||||
return Option( | |||||
"--trusted-host", | |||||
dest="trusted_hosts", | |||||
action="append", | |||||
metavar="HOSTNAME", | |||||
default=[], | |||||
help="Mark this host as trusted, even though it does not have valid " | |||||
"or any HTTPS.", | |||||
) | |||||
# Remove after 1.5 | |||||
process_dependency_links = partial( | |||||
Option, | |||||
"--process-dependency-links", | |||||
dest="process_dependency_links", | |||||
action="store_true", | |||||
default=False, | |||||
help="Enable the processing of dependency links.", | |||||
) # type: Any | |||||
def constraints(): | |||||
return Option( | |||||
'-c', '--constraint', | |||||
dest='constraints', | |||||
action='append', | |||||
default=[], | |||||
metavar='file', | |||||
help='Constrain versions using the given constraints file. ' | |||||
'This option can be used multiple times.' | |||||
) | |||||
def requirements(): | |||||
return Option( | |||||
'-r', '--requirement', | |||||
dest='requirements', | |||||
action='append', | |||||
default=[], | |||||
metavar='file', | |||||
help='Install from the given requirements file. ' | |||||
'This option can be used multiple times.' | |||||
) | |||||
def editable(): | |||||
return Option( | |||||
'-e', '--editable', | |||||
dest='editables', | |||||
action='append', | |||||
default=[], | |||||
metavar='path/url', | |||||
help=('Install a project in editable mode (i.e. setuptools ' | |||||
'"develop mode") from a local project path or a VCS url.'), | |||||
) | |||||
src = partial( | |||||
Option, | |||||
'--src', '--source', '--source-dir', '--source-directory', | |||||
dest='src_dir', | |||||
metavar='dir', | |||||
default=src_prefix, | |||||
help='Directory to check out editable projects into. ' | |||||
'The default in a virtualenv is "<venv path>/src". ' | |||||
'The default for global installs is "<current dir>/src".' | |||||
) # type: Any | |||||
def _get_format_control(values, option): | |||||
"""Get a format_control object.""" | |||||
return getattr(values, option.dest) | |||||
def _handle_no_binary(option, opt_str, value, parser): | |||||
existing = getattr(parser.values, option.dest) | |||||
fmt_ctl_handle_mutual_exclude( | |||||
value, existing.no_binary, existing.only_binary, | |||||
) | |||||
def _handle_only_binary(option, opt_str, value, parser): | |||||
existing = getattr(parser.values, option.dest) | |||||
fmt_ctl_handle_mutual_exclude( | |||||
value, existing.only_binary, existing.no_binary, | |||||
) | |||||
def no_binary(): | |||||
return Option( | |||||
"--no-binary", dest="format_control", action="callback", | |||||
callback=_handle_no_binary, type="str", | |||||
default=FormatControl(set(), set()), | |||||
help="Do not use binary packages. Can be supplied multiple times, and " | |||||
"each time adds to the existing value. Accepts either :all: to " | |||||
"disable all binary packages, :none: to empty the set, or one or " | |||||
"more package names with commas between them. Note that some " | |||||
"packages are tricky to compile and may fail to install when " | |||||
"this option is used on them.", | |||||
) | |||||
def only_binary(): | |||||
return Option( | |||||
"--only-binary", dest="format_control", action="callback", | |||||
callback=_handle_only_binary, type="str", | |||||
default=FormatControl(set(), set()), | |||||
help="Do not use source packages. Can be supplied multiple times, and " | |||||
"each time adds to the existing value. Accepts either :all: to " | |||||
"disable all source packages, :none: to empty the set, or one or " | |||||
"more package names with commas between them. Packages without " | |||||
"binary distributions will fail to install when this option is " | |||||
"used on them.", | |||||
) | |||||
cache_dir = partial( | |||||
Option, | |||||
"--cache-dir", | |||||
dest="cache_dir", | |||||
default=USER_CACHE_DIR, | |||||
metavar="dir", | |||||
help="Store the cache data in <dir>." | |||||
) | |||||
no_cache = partial( | |||||
Option, | |||||
"--no-cache-dir", | |||||
dest="cache_dir", | |||||
action="store_false", | |||||
help="Disable the cache.", | |||||
) | |||||
no_deps = partial( | |||||
Option, | |||||
'--no-deps', '--no-dependencies', | |||||
dest='ignore_dependencies', | |||||
action='store_true', | |||||
default=False, | |||||
help="Don't install package dependencies.", | |||||
) # type: Any | |||||
build_dir = partial( | |||||
Option, | |||||
'-b', '--build', '--build-dir', '--build-directory', | |||||
dest='build_dir', | |||||
metavar='dir', | |||||
help='Directory to unpack packages into and build in. Note that ' | |||||
'an initial build still takes place in a temporary directory. ' | |||||
'The location of temporary directories can be controlled by setting ' | |||||
'the TMPDIR environment variable (TEMP on Windows) appropriately. ' | |||||
'When passed, build directories are not cleaned in case of failures.' | |||||
) # type: Any | |||||
ignore_requires_python = partial( | |||||
Option, | |||||
'--ignore-requires-python', | |||||
dest='ignore_requires_python', | |||||
action='store_true', | |||||
help='Ignore the Requires-Python information.' | |||||
) # type: Any | |||||
no_build_isolation = partial( | |||||
Option, | |||||
'--no-build-isolation', | |||||
dest='build_isolation', | |||||
action='store_false', | |||||
default=True, | |||||
help='Disable isolation when building a modern source distribution. ' | |||||
'Build dependencies specified by PEP 518 must be already installed ' | |||||
'if this option is used.' | |||||
) # type: Any | |||||
install_options = partial( | |||||
Option, | |||||
'--install-option', | |||||
dest='install_options', | |||||
action='append', | |||||
metavar='options', | |||||
help="Extra arguments to be supplied to the setup.py install " | |||||
"command (use like --install-option=\"--install-scripts=/usr/local/" | |||||
"bin\"). Use multiple --install-option options to pass multiple " | |||||
"options to setup.py install. If you are using an option with a " | |||||
"directory path, be sure to use absolute path.", | |||||
) # type: Any | |||||
global_options = partial( | |||||
Option, | |||||
'--global-option', | |||||
dest='global_options', | |||||
action='append', | |||||
metavar='options', | |||||
help="Extra global options to be supplied to the setup.py " | |||||
"call before the install command.", | |||||
) # type: Any | |||||
no_clean = partial( | |||||
Option, | |||||
'--no-clean', | |||||
action='store_true', | |||||
default=False, | |||||
help="Don't clean up build directories)." | |||||
) # type: Any | |||||
pre = partial( | |||||
Option, | |||||
'--pre', | |||||
action='store_true', | |||||
default=False, | |||||
help="Include pre-release and development versions. By default, " | |||||
"pip only finds stable versions.", | |||||
) # type: Any | |||||
disable_pip_version_check = partial( | |||||
Option, | |||||
"--disable-pip-version-check", | |||||
dest="disable_pip_version_check", | |||||
action="store_true", | |||||
default=False, | |||||
help="Don't periodically check PyPI to determine whether a new version " | |||||
"of pip is available for download. Implied with --no-index.", | |||||
) # type: Any | |||||
# Deprecated, Remove later | |||||
always_unzip = partial( | |||||
Option, | |||||
'-Z', '--always-unzip', | |||||
dest='always_unzip', | |||||
action='store_true', | |||||
help=SUPPRESS_HELP, | |||||
) # type: Any | |||||
def _merge_hash(option, opt_str, value, parser): | |||||
"""Given a value spelled "algo:digest", append the digest to a list | |||||
pointed to in a dict by the algo name.""" | |||||
if not parser.values.hashes: | |||||
parser.values.hashes = {} | |||||
try: | |||||
algo, digest = value.split(':', 1) | |||||
except ValueError: | |||||
parser.error('Arguments to %s must be a hash name ' | |||||
'followed by a value, like --hash=sha256:abcde...' % | |||||
opt_str) | |||||
if algo not in STRONG_HASHES: | |||||
parser.error('Allowed hash algorithms for %s are %s.' % | |||||
(opt_str, ', '.join(STRONG_HASHES))) | |||||
parser.values.hashes.setdefault(algo, []).append(digest) | |||||
hash = partial( | |||||
Option, | |||||
'--hash', | |||||
# Hash values eventually end up in InstallRequirement.hashes due to | |||||
# __dict__ copying in process_line(). | |||||
dest='hashes', | |||||
action='callback', | |||||
callback=_merge_hash, | |||||
type='string', | |||||
help="Verify that the package's archive matches this " | |||||
'hash before installing. Example: --hash=sha256:abcdef...', | |||||
) # type: Any | |||||
require_hashes = partial( | |||||
Option, | |||||
'--require-hashes', | |||||
dest='require_hashes', | |||||
action='store_true', | |||||
default=False, | |||||
help='Require a hash to check each requirement against, for ' | |||||
'repeatable installs. This option is implied when any package in a ' | |||||
'requirements file has a --hash option.', | |||||
) # type: Any | |||||
########## | |||||
# groups # | |||||
########## | |||||
general_group = { | |||||
'name': 'General Options', | |||||
'options': [ | |||||
help_, | |||||
isolated_mode, | |||||
require_virtualenv, | |||||
verbose, | |||||
version, | |||||
quiet, | |||||
log, | |||||
no_input, | |||||
proxy, | |||||
retries, | |||||
timeout, | |||||
skip_requirements_regex, | |||||
exists_action, | |||||
trusted_host, | |||||
cert, | |||||
client_cert, | |||||
cache_dir, | |||||
no_cache, | |||||
disable_pip_version_check, | |||||
no_color, | |||||
] | |||||
} | |||||
index_group = { | |||||
'name': 'Package Index Options', | |||||
'options': [ | |||||
index_url, | |||||
extra_index_url, | |||||
no_index, | |||||
find_links, | |||||
process_dependency_links, | |||||
] | |||||
} |
""" | |||||
Package containing all pip commands | |||||
""" | |||||
from __future__ import absolute_import | |||||
from pip._internal.commands.completion import CompletionCommand | |||||
from pip._internal.commands.configuration import ConfigurationCommand | |||||
from pip._internal.commands.download import DownloadCommand | |||||
from pip._internal.commands.freeze import FreezeCommand | |||||
from pip._internal.commands.hash import HashCommand | |||||
from pip._internal.commands.help import HelpCommand | |||||
from pip._internal.commands.list import ListCommand | |||||
from pip._internal.commands.check import CheckCommand | |||||
from pip._internal.commands.search import SearchCommand | |||||
from pip._internal.commands.show import ShowCommand | |||||
from pip._internal.commands.install import InstallCommand | |||||
from pip._internal.commands.uninstall import UninstallCommand | |||||
from pip._internal.commands.wheel import WheelCommand | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import List, Type | |||||
from pip._internal.basecommand import Command | |||||
commands_order = [ | |||||
InstallCommand, | |||||
DownloadCommand, | |||||
UninstallCommand, | |||||
FreezeCommand, | |||||
ListCommand, | |||||
ShowCommand, | |||||
CheckCommand, | |||||
ConfigurationCommand, | |||||
SearchCommand, | |||||
WheelCommand, | |||||
HashCommand, | |||||
CompletionCommand, | |||||
HelpCommand, | |||||
] # type: List[Type[Command]] | |||||
commands_dict = {c.name: c for c in commands_order} | |||||
def get_summaries(ordered=True): | |||||
"""Yields sorted (command name, command summary) tuples.""" | |||||
if ordered: | |||||
cmditems = _sort_commands(commands_dict, commands_order) | |||||
else: | |||||
cmditems = commands_dict.items() | |||||
for name, command_class in cmditems: | |||||
yield (name, command_class.summary) | |||||
def get_similar_commands(name): | |||||
"""Command name auto-correct.""" | |||||
from difflib import get_close_matches | |||||
name = name.lower() | |||||
close_commands = get_close_matches(name, commands_dict.keys()) | |||||
if close_commands: | |||||
return close_commands[0] | |||||
else: | |||||
return False | |||||
def _sort_commands(cmddict, order): | |||||
def keyfn(key): | |||||
try: | |||||
return order.index(key[1]) | |||||
except ValueError: | |||||
# unordered items should come last | |||||
return 0xff | |||||
return sorted(cmddict.items(), key=keyfn) |
import logging | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.operations.check import ( | |||||
check_package_set, create_package_set_from_installed, | |||||
) | |||||
from pip._internal.utils.misc import get_installed_distributions | |||||
logger = logging.getLogger(__name__) | |||||
class CheckCommand(Command): | |||||
"""Verify installed packages have compatible dependencies.""" | |||||
name = 'check' | |||||
usage = """ | |||||
%prog [options]""" | |||||
summary = 'Verify installed packages have compatible dependencies.' | |||||
def run(self, options, args): | |||||
package_set = create_package_set_from_installed() | |||||
missing, conflicting = check_package_set(package_set) | |||||
for project_name in missing: | |||||
version = package_set[project_name].version | |||||
for dependency in missing[project_name]: | |||||
logger.info( | |||||
"%s %s requires %s, which is not installed.", | |||||
project_name, version, dependency[0], | |||||
) | |||||
for project_name in conflicting: | |||||
version = package_set[project_name].version | |||||
for dep_name, dep_version, req in conflicting[project_name]: | |||||
logger.info( | |||||
"%s %s has requirement %s, but you have %s %s.", | |||||
project_name, version, req, dep_name, dep_version, | |||||
) | |||||
if missing or conflicting: | |||||
return 1 | |||||
else: | |||||
logger.info("No broken requirements found.") |
from __future__ import absolute_import | |||||
import sys | |||||
import textwrap | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.utils.misc import get_prog | |||||
BASE_COMPLETION = """ | |||||
# pip %(shell)s completion start%(script)s# pip %(shell)s completion end | |||||
""" | |||||
COMPLETION_SCRIPTS = { | |||||
'bash': """ | |||||
_pip_completion() | |||||
{ | |||||
COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ | |||||
COMP_CWORD=$COMP_CWORD \\ | |||||
PIP_AUTO_COMPLETE=1 $1 ) ) | |||||
} | |||||
complete -o default -F _pip_completion %(prog)s | |||||
""", | |||||
'zsh': """ | |||||
function _pip_completion { | |||||
local words cword | |||||
read -Ac words | |||||
read -cn cword | |||||
reply=( $( COMP_WORDS="$words[*]" \\ | |||||
COMP_CWORD=$(( cword-1 )) \\ | |||||
PIP_AUTO_COMPLETE=1 $words[1] ) ) | |||||
} | |||||
compctl -K _pip_completion %(prog)s | |||||
""", | |||||
'fish': """ | |||||
function __fish_complete_pip | |||||
set -lx COMP_WORDS (commandline -o) "" | |||||
set -lx COMP_CWORD ( \\ | |||||
math (contains -i -- (commandline -t) $COMP_WORDS)-1 \\ | |||||
) | |||||
set -lx PIP_AUTO_COMPLETE 1 | |||||
string split \\ -- (eval $COMP_WORDS[1]) | |||||
end | |||||
complete -fa "(__fish_complete_pip)" -c %(prog)s | |||||
""", | |||||
} | |||||
class CompletionCommand(Command): | |||||
"""A helper command to be used for command completion.""" | |||||
name = 'completion' | |||||
summary = 'A helper command used for command completion.' | |||||
ignore_require_venv = True | |||||
def __init__(self, *args, **kw): | |||||
super(CompletionCommand, self).__init__(*args, **kw) | |||||
cmd_opts = self.cmd_opts | |||||
cmd_opts.add_option( | |||||
'--bash', '-b', | |||||
action='store_const', | |||||
const='bash', | |||||
dest='shell', | |||||
help='Emit completion code for bash') | |||||
cmd_opts.add_option( | |||||
'--zsh', '-z', | |||||
action='store_const', | |||||
const='zsh', | |||||
dest='shell', | |||||
help='Emit completion code for zsh') | |||||
cmd_opts.add_option( | |||||
'--fish', '-f', | |||||
action='store_const', | |||||
const='fish', | |||||
dest='shell', | |||||
help='Emit completion code for fish') | |||||
self.parser.insert_option_group(0, cmd_opts) | |||||
def run(self, options, args): | |||||
"""Prints the completion code of the given shell""" | |||||
shells = COMPLETION_SCRIPTS.keys() | |||||
shell_options = ['--' + shell for shell in sorted(shells)] | |||||
if options.shell in shells: | |||||
script = textwrap.dedent( | |||||
COMPLETION_SCRIPTS.get(options.shell, '') % { | |||||
'prog': get_prog(), | |||||
} | |||||
) | |||||
print(BASE_COMPLETION % {'script': script, 'shell': options.shell}) | |||||
else: | |||||
sys.stderr.write( | |||||
'ERROR: You must pass %s\n' % ' or '.join(shell_options) | |||||
) |
import logging | |||||
import os | |||||
import subprocess | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.configuration import Configuration, kinds | |||||
from pip._internal.exceptions import PipError | |||||
from pip._internal.locations import venv_config_file | |||||
from pip._internal.status_codes import ERROR, SUCCESS | |||||
from pip._internal.utils.misc import get_prog | |||||
logger = logging.getLogger(__name__) | |||||
class ConfigurationCommand(Command): | |||||
"""Manage local and global configuration. | |||||
Subcommands: | |||||
list: List the active configuration (or from the file specified) | |||||
edit: Edit the configuration file in an editor | |||||
get: Get the value associated with name | |||||
set: Set the name=value | |||||
unset: Unset the value associated with name | |||||
If none of --user, --global and --venv are passed, a virtual | |||||
environment configuration file is used if one is active and the file | |||||
exists. Otherwise, all modifications happen on the to the user file by | |||||
default. | |||||
""" | |||||
name = 'config' | |||||
usage = """ | |||||
%prog [<file-option>] list | |||||
%prog [<file-option>] [--editor <editor-path>] edit | |||||
%prog [<file-option>] get name | |||||
%prog [<file-option>] set name value | |||||
%prog [<file-option>] unset name | |||||
""" | |||||
summary = "Manage local and global configuration." | |||||
def __init__(self, *args, **kwargs): | |||||
super(ConfigurationCommand, self).__init__(*args, **kwargs) | |||||
self.configuration = None | |||||
self.cmd_opts.add_option( | |||||
'--editor', | |||||
dest='editor', | |||||
action='store', | |||||
default=None, | |||||
help=( | |||||
'Editor to use to edit the file. Uses VISUAL or EDITOR ' | |||||
'environment variables if not provided.' | |||||
) | |||||
) | |||||
self.cmd_opts.add_option( | |||||
'--global', | |||||
dest='global_file', | |||||
action='store_true', | |||||
default=False, | |||||
help='Use the system-wide configuration file only' | |||||
) | |||||
self.cmd_opts.add_option( | |||||
'--user', | |||||
dest='user_file', | |||||
action='store_true', | |||||
default=False, | |||||
help='Use the user configuration file only' | |||||
) | |||||
self.cmd_opts.add_option( | |||||
'--venv', | |||||
dest='venv_file', | |||||
action='store_true', | |||||
default=False, | |||||
help='Use the virtualenv configuration file only' | |||||
) | |||||
self.parser.insert_option_group(0, self.cmd_opts) | |||||
def run(self, options, args): | |||||
handlers = { | |||||
"list": self.list_values, | |||||
"edit": self.open_in_editor, | |||||
"get": self.get_name, | |||||
"set": self.set_name_value, | |||||
"unset": self.unset_name | |||||
} | |||||
# Determine action | |||||
if not args or args[0] not in handlers: | |||||
logger.error("Need an action ({}) to perform.".format( | |||||
", ".join(sorted(handlers))) | |||||
) | |||||
return ERROR | |||||
action = args[0] | |||||
# Determine which configuration files are to be loaded | |||||
# Depends on whether the command is modifying. | |||||
try: | |||||
load_only = self._determine_file( | |||||
options, need_value=(action in ["get", "set", "unset", "edit"]) | |||||
) | |||||
except PipError as e: | |||||
logger.error(e.args[0]) | |||||
return ERROR | |||||
# Load a new configuration | |||||
self.configuration = Configuration( | |||||
isolated=options.isolated_mode, load_only=load_only | |||||
) | |||||
self.configuration.load() | |||||
# Error handling happens here, not in the action-handlers. | |||||
try: | |||||
handlers[action](options, args[1:]) | |||||
except PipError as e: | |||||
logger.error(e.args[0]) | |||||
return ERROR | |||||
return SUCCESS | |||||
def _determine_file(self, options, need_value): | |||||
file_options = { | |||||
kinds.USER: options.user_file, | |||||
kinds.GLOBAL: options.global_file, | |||||
kinds.VENV: options.venv_file | |||||
} | |||||
if sum(file_options.values()) == 0: | |||||
if not need_value: | |||||
return None | |||||
# Default to user, unless there's a virtualenv file. | |||||
elif os.path.exists(venv_config_file): | |||||
return kinds.VENV | |||||
else: | |||||
return kinds.USER | |||||
elif sum(file_options.values()) == 1: | |||||
# There's probably a better expression for this. | |||||
return [key for key in file_options if file_options[key]][0] | |||||
raise PipError( | |||||
"Need exactly one file to operate upon " | |||||
"(--user, --venv, --global) to perform." | |||||
) | |||||
def list_values(self, options, args): | |||||
self._get_n_args(args, "list", n=0) | |||||
for key, value in sorted(self.configuration.items()): | |||||
logger.info("%s=%r", key, value) | |||||
def get_name(self, options, args): | |||||
key = self._get_n_args(args, "get [name]", n=1) | |||||
value = self.configuration.get_value(key) | |||||
logger.info("%s", value) | |||||
def set_name_value(self, options, args): | |||||
key, value = self._get_n_args(args, "set [name] [value]", n=2) | |||||
self.configuration.set_value(key, value) | |||||
self._save_configuration() | |||||
def unset_name(self, options, args): | |||||
key = self._get_n_args(args, "unset [name]", n=1) | |||||
self.configuration.unset_value(key) | |||||
self._save_configuration() | |||||
def open_in_editor(self, options, args): | |||||
editor = self._determine_editor(options) | |||||
fname = self.configuration.get_file_to_edit() | |||||
if fname is None: | |||||
raise PipError("Could not determine appropriate file.") | |||||
try: | |||||
subprocess.check_call([editor, fname]) | |||||
except subprocess.CalledProcessError as e: | |||||
raise PipError( | |||||
"Editor Subprocess exited with exit code {}" | |||||
.format(e.returncode) | |||||
) | |||||
def _get_n_args(self, args, example, n): | |||||
"""Helper to make sure the command got the right number of arguments | |||||
""" | |||||
if len(args) != n: | |||||
msg = ( | |||||
'Got unexpected number of arguments, expected {}. ' | |||||
'(example: "{} config {}")' | |||||
).format(n, get_prog(), example) | |||||
raise PipError(msg) | |||||
if n == 1: | |||||
return args[0] | |||||
else: | |||||
return args | |||||
def _save_configuration(self): | |||||
# We successfully ran a modifying command. Need to save the | |||||
# configuration. | |||||
try: | |||||
self.configuration.save() | |||||
except Exception: | |||||
logger.error( | |||||
"Unable to save configuration. Please report this as a bug.", | |||||
exc_info=1 | |||||
) | |||||
raise PipError("Internal Error.") | |||||
def _determine_editor(self, options): | |||||
if options.editor is not None: | |||||
return options.editor | |||||
elif "VISUAL" in os.environ: | |||||
return os.environ["VISUAL"] | |||||
elif "EDITOR" in os.environ: | |||||
return os.environ["EDITOR"] | |||||
else: | |||||
raise PipError("Could not determine editor to use.") |
from __future__ import absolute_import | |||||
import logging | |||||
import os | |||||
from pip._internal import cmdoptions | |||||
from pip._internal.basecommand import RequirementCommand | |||||
from pip._internal.exceptions import CommandError | |||||
from pip._internal.index import FormatControl | |||||
from pip._internal.operations.prepare import RequirementPreparer | |||||
from pip._internal.req import RequirementSet | |||||
from pip._internal.resolve import Resolver | |||||
from pip._internal.utils.filesystem import check_path_owner | |||||
from pip._internal.utils.misc import ensure_dir, normalize_path | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
logger = logging.getLogger(__name__) | |||||
class DownloadCommand(RequirementCommand): | |||||
""" | |||||
Download packages from: | |||||
- PyPI (and other indexes) using requirement specifiers. | |||||
- VCS project urls. | |||||
- Local project directories. | |||||
- Local or remote source archives. | |||||
pip also supports downloading from "requirements files", which provide | |||||
an easy way to specify a whole environment to be downloaded. | |||||
""" | |||||
name = 'download' | |||||
usage = """ | |||||
%prog [options] <requirement specifier> [package-index-options] ... | |||||
%prog [options] -r <requirements file> [package-index-options] ... | |||||
%prog [options] <vcs project url> ... | |||||
%prog [options] <local project path> ... | |||||
%prog [options] <archive url/path> ...""" | |||||
summary = 'Download packages.' | |||||
def __init__(self, *args, **kw): | |||||
super(DownloadCommand, self).__init__(*args, **kw) | |||||
cmd_opts = self.cmd_opts | |||||
cmd_opts.add_option(cmdoptions.constraints()) | |||||
cmd_opts.add_option(cmdoptions.requirements()) | |||||
cmd_opts.add_option(cmdoptions.build_dir()) | |||||
cmd_opts.add_option(cmdoptions.no_deps()) | |||||
cmd_opts.add_option(cmdoptions.global_options()) | |||||
cmd_opts.add_option(cmdoptions.no_binary()) | |||||
cmd_opts.add_option(cmdoptions.only_binary()) | |||||
cmd_opts.add_option(cmdoptions.src()) | |||||
cmd_opts.add_option(cmdoptions.pre()) | |||||
cmd_opts.add_option(cmdoptions.no_clean()) | |||||
cmd_opts.add_option(cmdoptions.require_hashes()) | |||||
cmd_opts.add_option(cmdoptions.progress_bar()) | |||||
cmd_opts.add_option(cmdoptions.no_build_isolation()) | |||||
cmd_opts.add_option( | |||||
'-d', '--dest', '--destination-dir', '--destination-directory', | |||||
dest='download_dir', | |||||
metavar='dir', | |||||
default=os.curdir, | |||||
help=("Download packages into <dir>."), | |||||
) | |||||
cmd_opts.add_option( | |||||
'--platform', | |||||
dest='platform', | |||||
metavar='platform', | |||||
default=None, | |||||
help=("Only download wheels compatible with <platform>. " | |||||
"Defaults to the platform of the running system."), | |||||
) | |||||
cmd_opts.add_option( | |||||
'--python-version', | |||||
dest='python_version', | |||||
metavar='python_version', | |||||
default=None, | |||||
help=("Only download wheels compatible with Python " | |||||
"interpreter version <version>. If not specified, then the " | |||||
"current system interpreter minor version is used. A major " | |||||
"version (e.g. '2') can be specified to match all " | |||||
"minor revs of that major version. A minor version " | |||||
"(e.g. '34') can also be specified."), | |||||
) | |||||
cmd_opts.add_option( | |||||
'--implementation', | |||||
dest='implementation', | |||||
metavar='implementation', | |||||
default=None, | |||||
help=("Only download wheels compatible with Python " | |||||
"implementation <implementation>, e.g. 'pp', 'jy', 'cp', " | |||||
" or 'ip'. If not specified, then the current " | |||||
"interpreter implementation is used. Use 'py' to force " | |||||
"implementation-agnostic wheels."), | |||||
) | |||||
cmd_opts.add_option( | |||||
'--abi', | |||||
dest='abi', | |||||
metavar='abi', | |||||
default=None, | |||||
help=("Only download wheels compatible with Python " | |||||
"abi <abi>, e.g. 'pypy_41'. If not specified, then the " | |||||
"current interpreter abi tag is used. Generally " | |||||
"you will need to specify --implementation, " | |||||
"--platform, and --python-version when using " | |||||
"this option."), | |||||
) | |||||
index_opts = cmdoptions.make_option_group( | |||||
cmdoptions.index_group, | |||||
self.parser, | |||||
) | |||||
self.parser.insert_option_group(0, index_opts) | |||||
self.parser.insert_option_group(0, cmd_opts) | |||||
def run(self, options, args): | |||||
options.ignore_installed = True | |||||
# editable doesn't really make sense for `pip download`, but the bowels | |||||
# of the RequirementSet code require that property. | |||||
options.editables = [] | |||||
if options.python_version: | |||||
python_versions = [options.python_version] | |||||
else: | |||||
python_versions = None | |||||
dist_restriction_set = any([ | |||||
options.python_version, | |||||
options.platform, | |||||
options.abi, | |||||
options.implementation, | |||||
]) | |||||
binary_only = FormatControl(set(), {':all:'}) | |||||
no_sdist_dependencies = ( | |||||
options.format_control != binary_only and | |||||
not options.ignore_dependencies | |||||
) | |||||
if dist_restriction_set and no_sdist_dependencies: | |||||
raise CommandError( | |||||
"When restricting platform and interpreter constraints using " | |||||
"--python-version, --platform, --abi, or --implementation, " | |||||
"either --no-deps must be set, or --only-binary=:all: must be " | |||||
"set and --no-binary must not be set (or must be set to " | |||||
":none:)." | |||||
) | |||||
options.src_dir = os.path.abspath(options.src_dir) | |||||
options.download_dir = normalize_path(options.download_dir) | |||||
ensure_dir(options.download_dir) | |||||
with self._build_session(options) as session: | |||||
finder = self._build_package_finder( | |||||
options=options, | |||||
session=session, | |||||
platform=options.platform, | |||||
python_versions=python_versions, | |||||
abi=options.abi, | |||||
implementation=options.implementation, | |||||
) | |||||
build_delete = (not (options.no_clean or options.build_dir)) | |||||
if options.cache_dir and not check_path_owner(options.cache_dir): | |||||
logger.warning( | |||||
"The directory '%s' or its parent directory is not owned " | |||||
"by the current user and caching wheels has been " | |||||
"disabled. check the permissions and owner of that " | |||||
"directory. If executing pip with sudo, you may want " | |||||
"sudo's -H flag.", | |||||
options.cache_dir, | |||||
) | |||||
options.cache_dir = None | |||||
with TempDirectory( | |||||
options.build_dir, delete=build_delete, kind="download" | |||||
) as directory: | |||||
requirement_set = RequirementSet( | |||||
require_hashes=options.require_hashes, | |||||
) | |||||
self.populate_requirement_set( | |||||
requirement_set, | |||||
args, | |||||
options, | |||||
finder, | |||||
session, | |||||
self.name, | |||||
None | |||||
) | |||||
preparer = RequirementPreparer( | |||||
build_dir=directory.path, | |||||
src_dir=options.src_dir, | |||||
download_dir=options.download_dir, | |||||
wheel_download_dir=None, | |||||
progress_bar=options.progress_bar, | |||||
build_isolation=options.build_isolation, | |||||
) | |||||
resolver = Resolver( | |||||
preparer=preparer, | |||||
finder=finder, | |||||
session=session, | |||||
wheel_cache=None, | |||||
use_user_site=False, | |||||
upgrade_strategy="to-satisfy-only", | |||||
force_reinstall=False, | |||||
ignore_dependencies=options.ignore_dependencies, | |||||
ignore_requires_python=False, | |||||
ignore_installed=True, | |||||
isolated=options.isolated_mode, | |||||
) | |||||
resolver.resolve(requirement_set) | |||||
downloaded = ' '.join([ | |||||
req.name for req in requirement_set.successfully_downloaded | |||||
]) | |||||
if downloaded: | |||||
logger.info('Successfully downloaded %s', downloaded) | |||||
# Clean up | |||||
if not options.no_clean: | |||||
requirement_set.cleanup_files() | |||||
return requirement_set |
from __future__ import absolute_import | |||||
import sys | |||||
from pip._internal import index | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.cache import WheelCache | |||||
from pip._internal.compat import stdlib_pkgs | |||||
from pip._internal.operations.freeze import freeze | |||||
DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'} | |||||
class FreezeCommand(Command): | |||||
""" | |||||
Output installed packages in requirements format. | |||||
packages are listed in a case-insensitive sorted order. | |||||
""" | |||||
name = 'freeze' | |||||
usage = """ | |||||
%prog [options]""" | |||||
summary = 'Output installed packages in requirements format.' | |||||
log_streams = ("ext://sys.stderr", "ext://sys.stderr") | |||||
def __init__(self, *args, **kw): | |||||
super(FreezeCommand, self).__init__(*args, **kw) | |||||
self.cmd_opts.add_option( | |||||
'-r', '--requirement', | |||||
dest='requirements', | |||||
action='append', | |||||
default=[], | |||||
metavar='file', | |||||
help="Use the order in the given requirements file and its " | |||||
"comments when generating output. This option can be " | |||||
"used multiple times.") | |||||
self.cmd_opts.add_option( | |||||
'-f', '--find-links', | |||||
dest='find_links', | |||||
action='append', | |||||
default=[], | |||||
metavar='URL', | |||||
help='URL for finding packages, which will be added to the ' | |||||
'output.') | |||||
self.cmd_opts.add_option( | |||||
'-l', '--local', | |||||
dest='local', | |||||
action='store_true', | |||||
default=False, | |||||
help='If in a virtualenv that has global access, do not output ' | |||||
'globally-installed packages.') | |||||
self.cmd_opts.add_option( | |||||
'--user', | |||||
dest='user', | |||||
action='store_true', | |||||
default=False, | |||||
help='Only output packages installed in user-site.') | |||||
self.cmd_opts.add_option( | |||||
'--all', | |||||
dest='freeze_all', | |||||
action='store_true', | |||||
help='Do not skip these packages in the output:' | |||||
' %s' % ', '.join(DEV_PKGS)) | |||||
self.cmd_opts.add_option( | |||||
'--exclude-editable', | |||||
dest='exclude_editable', | |||||
action='store_true', | |||||
help='Exclude editable package from output.') | |||||
self.parser.insert_option_group(0, self.cmd_opts) | |||||
def run(self, options, args): | |||||
format_control = index.FormatControl(set(), set()) | |||||
wheel_cache = WheelCache(options.cache_dir, format_control) | |||||
skip = set(stdlib_pkgs) | |||||
if not options.freeze_all: | |||||
skip.update(DEV_PKGS) | |||||
freeze_kwargs = dict( | |||||
requirement=options.requirements, | |||||
find_links=options.find_links, | |||||
local_only=options.local, | |||||
user_only=options.user, | |||||
skip_regex=options.skip_requirements_regex, | |||||
isolated=options.isolated_mode, | |||||
wheel_cache=wheel_cache, | |||||
skip=skip, | |||||
exclude_editable=options.exclude_editable, | |||||
) | |||||
try: | |||||
for line in freeze(**freeze_kwargs): | |||||
sys.stdout.write(line + '\n') | |||||
finally: | |||||
wheel_cache.cleanup() |
from __future__ import absolute_import | |||||
import hashlib | |||||
import logging | |||||
import sys | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.status_codes import ERROR | |||||
from pip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES | |||||
from pip._internal.utils.misc import read_chunks | |||||
logger = logging.getLogger(__name__) | |||||
class HashCommand(Command): | |||||
""" | |||||
Compute a hash of a local package archive. | |||||
These can be used with --hash in a requirements file to do repeatable | |||||
installs. | |||||
""" | |||||
name = 'hash' | |||||
usage = '%prog [options] <file> ...' | |||||
summary = 'Compute hashes of package archives.' | |||||
ignore_require_venv = True | |||||
def __init__(self, *args, **kw): | |||||
super(HashCommand, self).__init__(*args, **kw) | |||||
self.cmd_opts.add_option( | |||||
'-a', '--algorithm', | |||||
dest='algorithm', | |||||
choices=STRONG_HASHES, | |||||
action='store', | |||||
default=FAVORITE_HASH, | |||||
help='The hash algorithm to use: one of %s' % | |||||
', '.join(STRONG_HASHES)) | |||||
self.parser.insert_option_group(0, self.cmd_opts) | |||||
def run(self, options, args): | |||||
if not args: | |||||
self.parser.print_usage(sys.stderr) | |||||
return ERROR | |||||
algorithm = options.algorithm | |||||
for path in args: | |||||
logger.info('%s:\n--hash=%s:%s', | |||||
path, algorithm, _hash_of_file(path, algorithm)) | |||||
def _hash_of_file(path, algorithm): | |||||
"""Return the hash digest of a file.""" | |||||
with open(path, 'rb') as archive: | |||||
hash = hashlib.new(algorithm) | |||||
for chunk in read_chunks(archive): | |||||
hash.update(chunk) | |||||
return hash.hexdigest() |
from __future__ import absolute_import | |||||
from pip._internal.basecommand import SUCCESS, Command | |||||
from pip._internal.exceptions import CommandError | |||||
class HelpCommand(Command): | |||||
"""Show help for commands""" | |||||
name = 'help' | |||||
usage = """ | |||||
%prog <command>""" | |||||
summary = 'Show help for commands.' | |||||
ignore_require_venv = True | |||||
def run(self, options, args): | |||||
from pip._internal.commands import commands_dict, get_similar_commands | |||||
try: | |||||
# 'pip help' with no args is handled by pip.__init__.parseopt() | |||||
cmd_name = args[0] # the command we need help for | |||||
except IndexError: | |||||
return SUCCESS | |||||
if cmd_name not in commands_dict: | |||||
guess = get_similar_commands(cmd_name) | |||||
msg = ['unknown command "%s"' % cmd_name] | |||||
if guess: | |||||
msg.append('maybe you meant "%s"' % guess) | |||||
raise CommandError(' - '.join(msg)) | |||||
command = commands_dict[cmd_name]() | |||||
command.parser.print_help() | |||||
return SUCCESS |
from __future__ import absolute_import | |||||
import errno | |||||
import logging | |||||
import operator | |||||
import os | |||||
import shutil | |||||
from optparse import SUPPRESS_HELP | |||||
from pip._internal import cmdoptions | |||||
from pip._internal.basecommand import RequirementCommand | |||||
from pip._internal.cache import WheelCache | |||||
from pip._internal.exceptions import ( | |||||
CommandError, InstallationError, PreviousBuildDirError, | |||||
) | |||||
from pip._internal.locations import distutils_scheme, virtualenv_no_global | |||||
from pip._internal.operations.check import check_install_conflicts | |||||
from pip._internal.operations.prepare import RequirementPreparer | |||||
from pip._internal.req import RequirementSet, install_given_reqs | |||||
from pip._internal.resolve import Resolver | |||||
from pip._internal.status_codes import ERROR | |||||
from pip._internal.utils.filesystem import check_path_owner | |||||
from pip._internal.utils.misc import ensure_dir, get_installed_version | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.wheel import WheelBuilder | |||||
try: | |||||
import wheel | |||||
except ImportError: | |||||
wheel = None | |||||
logger = logging.getLogger(__name__) | |||||
class InstallCommand(RequirementCommand): | |||||
""" | |||||
Install packages from: | |||||
- PyPI (and other indexes) using requirement specifiers. | |||||
- VCS project urls. | |||||
- Local project directories. | |||||
- Local or remote source archives. | |||||
pip also supports installing from "requirements files", which provide | |||||
an easy way to specify a whole environment to be installed. | |||||
""" | |||||
name = 'install' | |||||
usage = """ | |||||
%prog [options] <requirement specifier> [package-index-options] ... | |||||
%prog [options] -r <requirements file> [package-index-options] ... | |||||
%prog [options] [-e] <vcs project url> ... | |||||
%prog [options] [-e] <local project path> ... | |||||
%prog [options] <archive url/path> ...""" | |||||
summary = 'Install packages.' | |||||
def __init__(self, *args, **kw): | |||||
super(InstallCommand, self).__init__(*args, **kw) | |||||
cmd_opts = self.cmd_opts | |||||
cmd_opts.add_option(cmdoptions.requirements()) | |||||
cmd_opts.add_option(cmdoptions.constraints()) | |||||
cmd_opts.add_option(cmdoptions.no_deps()) | |||||
cmd_opts.add_option(cmdoptions.pre()) | |||||
cmd_opts.add_option(cmdoptions.editable()) | |||||
cmd_opts.add_option( | |||||
'-t', '--target', | |||||
dest='target_dir', | |||||
metavar='dir', | |||||
default=None, | |||||
help='Install packages into <dir>. ' | |||||
'By default this will not replace existing files/folders in ' | |||||
'<dir>. Use --upgrade to replace existing packages in <dir> ' | |||||
'with new versions.' | |||||
) | |||||
cmd_opts.add_option( | |||||
'--user', | |||||
dest='use_user_site', | |||||
action='store_true', | |||||
help="Install to the Python user install directory for your " | |||||
"platform. Typically ~/.local/, or %APPDATA%\\Python on " | |||||
"Windows. (See the Python documentation for site.USER_BASE " | |||||
"for full details.)") | |||||
cmd_opts.add_option( | |||||
'--no-user', | |||||
dest='use_user_site', | |||||
action='store_false', | |||||
help=SUPPRESS_HELP) | |||||
cmd_opts.add_option( | |||||
'--root', | |||||
dest='root_path', | |||||
metavar='dir', | |||||
default=None, | |||||
help="Install everything relative to this alternate root " | |||||
"directory.") | |||||
cmd_opts.add_option( | |||||
'--prefix', | |||||
dest='prefix_path', | |||||
metavar='dir', | |||||
default=None, | |||||
help="Installation prefix where lib, bin and other top-level " | |||||
"folders are placed") | |||||
cmd_opts.add_option(cmdoptions.build_dir()) | |||||
cmd_opts.add_option(cmdoptions.src()) | |||||
cmd_opts.add_option( | |||||
'-U', '--upgrade', | |||||
dest='upgrade', | |||||
action='store_true', | |||||
help='Upgrade all specified packages to the newest available ' | |||||
'version. The handling of dependencies depends on the ' | |||||
'upgrade-strategy used.' | |||||
) | |||||
cmd_opts.add_option( | |||||
'--upgrade-strategy', | |||||
dest='upgrade_strategy', | |||||
default='only-if-needed', | |||||
choices=['only-if-needed', 'eager'], | |||||
help='Determines how dependency upgrading should be handled ' | |||||
'[default: %default]. ' | |||||
'"eager" - dependencies are upgraded regardless of ' | |||||
'whether the currently installed version satisfies the ' | |||||
'requirements of the upgraded package(s). ' | |||||
'"only-if-needed" - are upgraded only when they do not ' | |||||
'satisfy the requirements of the upgraded package(s).' | |||||
) | |||||
cmd_opts.add_option( | |||||
'--force-reinstall', | |||||
dest='force_reinstall', | |||||
action='store_true', | |||||
help='Reinstall all packages even if they are already ' | |||||
'up-to-date.') | |||||
cmd_opts.add_option( | |||||
'-I', '--ignore-installed', | |||||
dest='ignore_installed', | |||||
action='store_true', | |||||
help='Ignore the installed packages (reinstalling instead).') | |||||
cmd_opts.add_option(cmdoptions.ignore_requires_python()) | |||||
cmd_opts.add_option(cmdoptions.no_build_isolation()) | |||||
cmd_opts.add_option(cmdoptions.install_options()) | |||||
cmd_opts.add_option(cmdoptions.global_options()) | |||||
cmd_opts.add_option( | |||||
"--compile", | |||||
action="store_true", | |||||
dest="compile", | |||||
default=True, | |||||
help="Compile Python source files to bytecode", | |||||
) | |||||
cmd_opts.add_option( | |||||
"--no-compile", | |||||
action="store_false", | |||||
dest="compile", | |||||
help="Do not compile Python source files to bytecode", | |||||
) | |||||
cmd_opts.add_option( | |||||
"--no-warn-script-location", | |||||
action="store_false", | |||||
dest="warn_script_location", | |||||
default=True, | |||||
help="Do not warn when installing scripts outside PATH", | |||||
) | |||||
cmd_opts.add_option( | |||||
"--no-warn-conflicts", | |||||
action="store_false", | |||||
dest="warn_about_conflicts", | |||||
default=True, | |||||
help="Do not warn about broken dependencies", | |||||
) | |||||
cmd_opts.add_option(cmdoptions.no_binary()) | |||||
cmd_opts.add_option(cmdoptions.only_binary()) | |||||
cmd_opts.add_option(cmdoptions.no_clean()) | |||||
cmd_opts.add_option(cmdoptions.require_hashes()) | |||||
cmd_opts.add_option(cmdoptions.progress_bar()) | |||||
index_opts = cmdoptions.make_option_group( | |||||
cmdoptions.index_group, | |||||
self.parser, | |||||
) | |||||
self.parser.insert_option_group(0, index_opts) | |||||
self.parser.insert_option_group(0, cmd_opts) | |||||
def run(self, options, args): | |||||
cmdoptions.check_install_build_global(options) | |||||
upgrade_strategy = "to-satisfy-only" | |||||
if options.upgrade: | |||||
upgrade_strategy = options.upgrade_strategy | |||||
if options.build_dir: | |||||
options.build_dir = os.path.abspath(options.build_dir) | |||||
options.src_dir = os.path.abspath(options.src_dir) | |||||
install_options = options.install_options or [] | |||||
if options.use_user_site: | |||||
if options.prefix_path: | |||||
raise CommandError( | |||||
"Can not combine '--user' and '--prefix' as they imply " | |||||
"different installation locations" | |||||
) | |||||
if virtualenv_no_global(): | |||||
raise InstallationError( | |||||
"Can not perform a '--user' install. User site-packages " | |||||
"are not visible in this virtualenv." | |||||
) | |||||
install_options.append('--user') | |||||
install_options.append('--prefix=') | |||||
target_temp_dir = TempDirectory(kind="target") | |||||
if options.target_dir: | |||||
options.ignore_installed = True | |||||
options.target_dir = os.path.abspath(options.target_dir) | |||||
if (os.path.exists(options.target_dir) and not | |||||
os.path.isdir(options.target_dir)): | |||||
raise CommandError( | |||||
"Target path exists but is not a directory, will not " | |||||
"continue." | |||||
) | |||||
# Create a target directory for using with the target option | |||||
target_temp_dir.create() | |||||
install_options.append('--home=' + target_temp_dir.path) | |||||
global_options = options.global_options or [] | |||||
with self._build_session(options) as session: | |||||
finder = self._build_package_finder(options, session) | |||||
build_delete = (not (options.no_clean or options.build_dir)) | |||||
wheel_cache = WheelCache(options.cache_dir, options.format_control) | |||||
if options.cache_dir and not check_path_owner(options.cache_dir): | |||||
logger.warning( | |||||
"The directory '%s' or its parent directory is not owned " | |||||
"by the current user and caching wheels has been " | |||||
"disabled. check the permissions and owner of that " | |||||
"directory. If executing pip with sudo, you may want " | |||||
"sudo's -H flag.", | |||||
options.cache_dir, | |||||
) | |||||
options.cache_dir = None | |||||
with TempDirectory( | |||||
options.build_dir, delete=build_delete, kind="install" | |||||
) as directory: | |||||
requirement_set = RequirementSet( | |||||
require_hashes=options.require_hashes, | |||||
) | |||||
try: | |||||
self.populate_requirement_set( | |||||
requirement_set, args, options, finder, session, | |||||
self.name, wheel_cache | |||||
) | |||||
preparer = RequirementPreparer( | |||||
build_dir=directory.path, | |||||
src_dir=options.src_dir, | |||||
download_dir=None, | |||||
wheel_download_dir=None, | |||||
progress_bar=options.progress_bar, | |||||
build_isolation=options.build_isolation, | |||||
) | |||||
resolver = Resolver( | |||||
preparer=preparer, | |||||
finder=finder, | |||||
session=session, | |||||
wheel_cache=wheel_cache, | |||||
use_user_site=options.use_user_site, | |||||
upgrade_strategy=upgrade_strategy, | |||||
force_reinstall=options.force_reinstall, | |||||
ignore_dependencies=options.ignore_dependencies, | |||||
ignore_requires_python=options.ignore_requires_python, | |||||
ignore_installed=options.ignore_installed, | |||||
isolated=options.isolated_mode, | |||||
) | |||||
resolver.resolve(requirement_set) | |||||
# If caching is disabled or wheel is not installed don't | |||||
# try to build wheels. | |||||
if wheel and options.cache_dir: | |||||
# build wheels before install. | |||||
wb = WheelBuilder( | |||||
finder, preparer, wheel_cache, | |||||
build_options=[], global_options=[], | |||||
) | |||||
# Ignore the result: a failed wheel will be | |||||
# installed from the sdist/vcs whatever. | |||||
wb.build( | |||||
requirement_set.requirements.values(), | |||||
session=session, autobuilding=True | |||||
) | |||||
to_install = resolver.get_installation_order( | |||||
requirement_set | |||||
) | |||||
# Consistency Checking of the package set we're installing. | |||||
should_warn_about_conflicts = ( | |||||
not options.ignore_dependencies and | |||||
options.warn_about_conflicts | |||||
) | |||||
if should_warn_about_conflicts: | |||||
self._warn_about_conflicts(to_install) | |||||
# Don't warn about script install locations if | |||||
# --target has been specified | |||||
warn_script_location = options.warn_script_location | |||||
if options.target_dir: | |||||
warn_script_location = False | |||||
installed = install_given_reqs( | |||||
to_install, | |||||
install_options, | |||||
global_options, | |||||
root=options.root_path, | |||||
home=target_temp_dir.path, | |||||
prefix=options.prefix_path, | |||||
pycompile=options.compile, | |||||
warn_script_location=warn_script_location, | |||||
use_user_site=options.use_user_site, | |||||
) | |||||
possible_lib_locations = get_lib_location_guesses( | |||||
user=options.use_user_site, | |||||
home=target_temp_dir.path, | |||||
root=options.root_path, | |||||
prefix=options.prefix_path, | |||||
isolated=options.isolated_mode, | |||||
) | |||||
reqs = sorted(installed, key=operator.attrgetter('name')) | |||||
items = [] | |||||
for req in reqs: | |||||
item = req.name | |||||
try: | |||||
installed_version = get_installed_version( | |||||
req.name, possible_lib_locations | |||||
) | |||||
if installed_version: | |||||
item += '-' + installed_version | |||||
except Exception: | |||||
pass | |||||
items.append(item) | |||||
installed = ' '.join(items) | |||||
if installed: | |||||
logger.info('Successfully installed %s', installed) | |||||
except EnvironmentError as error: | |||||
show_traceback = (self.verbosity >= 1) | |||||
message = create_env_error_message( | |||||
error, show_traceback, options.use_user_site, | |||||
) | |||||
logger.error(message, exc_info=show_traceback) | |||||
return ERROR | |||||
except PreviousBuildDirError: | |||||
options.no_clean = True | |||||
raise | |||||
finally: | |||||
# Clean up | |||||
if not options.no_clean: | |||||
requirement_set.cleanup_files() | |||||
wheel_cache.cleanup() | |||||
if options.target_dir: | |||||
self._handle_target_dir( | |||||
options.target_dir, target_temp_dir, options.upgrade | |||||
) | |||||
return requirement_set | |||||
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): | |||||
ensure_dir(target_dir) | |||||
# Checking both purelib and platlib directories for installed | |||||
# packages to be moved to target directory | |||||
lib_dir_list = [] | |||||
with target_temp_dir: | |||||
# Checking both purelib and platlib directories for installed | |||||
# packages to be moved to target directory | |||||
scheme = distutils_scheme('', home=target_temp_dir.path) | |||||
purelib_dir = scheme['purelib'] | |||||
platlib_dir = scheme['platlib'] | |||||
data_dir = scheme['data'] | |||||
if os.path.exists(purelib_dir): | |||||
lib_dir_list.append(purelib_dir) | |||||
if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: | |||||
lib_dir_list.append(platlib_dir) | |||||
if os.path.exists(data_dir): | |||||
lib_dir_list.append(data_dir) | |||||
for lib_dir in lib_dir_list: | |||||
for item in os.listdir(lib_dir): | |||||
if lib_dir == data_dir: | |||||
ddir = os.path.join(data_dir, item) | |||||
if any(s.startswith(ddir) for s in lib_dir_list[:-1]): | |||||
continue | |||||
target_item_dir = os.path.join(target_dir, item) | |||||
if os.path.exists(target_item_dir): | |||||
if not upgrade: | |||||
logger.warning( | |||||
'Target directory %s already exists. Specify ' | |||||
'--upgrade to force replacement.', | |||||
target_item_dir | |||||
) | |||||
continue | |||||
if os.path.islink(target_item_dir): | |||||
logger.warning( | |||||
'Target directory %s already exists and is ' | |||||
'a link. Pip will not automatically replace ' | |||||
'links, please remove if replacement is ' | |||||
'desired.', | |||||
target_item_dir | |||||
) | |||||
continue | |||||
if os.path.isdir(target_item_dir): | |||||
shutil.rmtree(target_item_dir) | |||||
else: | |||||
os.remove(target_item_dir) | |||||
shutil.move( | |||||
os.path.join(lib_dir, item), | |||||
target_item_dir | |||||
) | |||||
def _warn_about_conflicts(self, to_install): | |||||
package_set, _dep_info = check_install_conflicts(to_install) | |||||
missing, conflicting = _dep_info | |||||
# NOTE: There is some duplication here from pip check | |||||
for project_name in missing: | |||||
version = package_set[project_name][0] | |||||
for dependency in missing[project_name]: | |||||
logger.critical( | |||||
"%s %s requires %s, which is not installed.", | |||||
project_name, version, dependency[1], | |||||
) | |||||
for project_name in conflicting: | |||||
version = package_set[project_name][0] | |||||
for dep_name, dep_version, req in conflicting[project_name]: | |||||
logger.critical( | |||||
"%s %s has requirement %s, but you'll have %s %s which is " | |||||
"incompatible.", | |||||
project_name, version, req, dep_name, dep_version, | |||||
) | |||||
def get_lib_location_guesses(*args, **kwargs): | |||||
scheme = distutils_scheme('', *args, **kwargs) | |||||
return [scheme['purelib'], scheme['platlib']] | |||||
def create_env_error_message(error, show_traceback, using_user_site): | |||||
"""Format an error message for an EnvironmentError | |||||
It may occur anytime during the execution of the install command. | |||||
""" | |||||
parts = [] | |||||
# Mention the error if we are not going to show a traceback | |||||
parts.append("Could not install packages due to an EnvironmentError") | |||||
if not show_traceback: | |||||
parts.append(": ") | |||||
parts.append(str(error)) | |||||
else: | |||||
parts.append(".") | |||||
# Spilt the error indication from a helper message (if any) | |||||
parts[-1] += "\n" | |||||
# Suggest useful actions to the user: | |||||
# (1) using user site-packages or (2) verifying the permissions | |||||
if error.errno == errno.EACCES: | |||||
user_option_part = "Consider using the `--user` option" | |||||
permissions_part = "Check the permissions" | |||||
if not using_user_site: | |||||
parts.extend([ | |||||
user_option_part, " or ", | |||||
permissions_part.lower(), | |||||
]) | |||||
else: | |||||
parts.append(permissions_part) | |||||
parts.append(".\n") | |||||
return "".join(parts).strip() + "\n" |
from __future__ import absolute_import | |||||
import json | |||||
import logging | |||||
import warnings | |||||
from pip._vendor import six | |||||
from pip._vendor.six.moves import zip_longest | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.cmdoptions import index_group, make_option_group | |||||
from pip._internal.exceptions import CommandError | |||||
from pip._internal.index import PackageFinder | |||||
from pip._internal.utils.deprecation import RemovedInPip11Warning | |||||
from pip._internal.utils.misc import ( | |||||
dist_is_editable, get_installed_distributions, | |||||
) | |||||
from pip._internal.utils.packaging import get_installer | |||||
logger = logging.getLogger(__name__) | |||||
class ListCommand(Command): | |||||
""" | |||||
List installed packages, including editables. | |||||
Packages are listed in a case-insensitive sorted order. | |||||
""" | |||||
name = 'list' | |||||
usage = """ | |||||
%prog [options]""" | |||||
summary = 'List installed packages.' | |||||
def __init__(self, *args, **kw): | |||||
super(ListCommand, self).__init__(*args, **kw) | |||||
cmd_opts = self.cmd_opts | |||||
cmd_opts.add_option( | |||||
'-o', '--outdated', | |||||
action='store_true', | |||||
default=False, | |||||
help='List outdated packages') | |||||
cmd_opts.add_option( | |||||
'-u', '--uptodate', | |||||
action='store_true', | |||||
default=False, | |||||
help='List uptodate packages') | |||||
cmd_opts.add_option( | |||||
'-e', '--editable', | |||||
action='store_true', | |||||
default=False, | |||||
help='List editable projects.') | |||||
cmd_opts.add_option( | |||||
'-l', '--local', | |||||
action='store_true', | |||||
default=False, | |||||
help=('If in a virtualenv that has global access, do not list ' | |||||
'globally-installed packages.'), | |||||
) | |||||
self.cmd_opts.add_option( | |||||
'--user', | |||||
dest='user', | |||||
action='store_true', | |||||
default=False, | |||||
help='Only output packages installed in user-site.') | |||||
cmd_opts.add_option( | |||||
'--pre', | |||||
action='store_true', | |||||
default=False, | |||||
help=("Include pre-release and development versions. By default, " | |||||
"pip only finds stable versions."), | |||||
) | |||||
cmd_opts.add_option( | |||||
'--format', | |||||
action='store', | |||||
dest='list_format', | |||||
default="columns", | |||||
choices=('legacy', 'columns', 'freeze', 'json'), | |||||
help="Select the output format among: columns (default), freeze, " | |||||
"json, or legacy.", | |||||
) | |||||
cmd_opts.add_option( | |||||
'--not-required', | |||||
action='store_true', | |||||
dest='not_required', | |||||
help="List packages that are not dependencies of " | |||||
"installed packages.", | |||||
) | |||||
cmd_opts.add_option( | |||||
'--exclude-editable', | |||||
action='store_false', | |||||
dest='include_editable', | |||||
help='Exclude editable package from output.', | |||||
) | |||||
cmd_opts.add_option( | |||||
'--include-editable', | |||||
action='store_true', | |||||
dest='include_editable', | |||||
help='Include editable package from output.', | |||||
default=True, | |||||
) | |||||
index_opts = make_option_group(index_group, self.parser) | |||||
self.parser.insert_option_group(0, index_opts) | |||||
self.parser.insert_option_group(0, cmd_opts) | |||||
def _build_package_finder(self, options, index_urls, session): | |||||
""" | |||||
Create a package finder appropriate to this list command. | |||||
""" | |||||
return PackageFinder( | |||||
find_links=options.find_links, | |||||
index_urls=index_urls, | |||||
allow_all_prereleases=options.pre, | |||||
trusted_hosts=options.trusted_hosts, | |||||
process_dependency_links=options.process_dependency_links, | |||||
session=session, | |||||
) | |||||
def run(self, options, args): | |||||
if options.list_format == "legacy": | |||||
warnings.warn( | |||||
"The legacy format has been deprecated and will be removed " | |||||
"in the future.", | |||||
RemovedInPip11Warning, | |||||
) | |||||
if options.outdated and options.uptodate: | |||||
raise CommandError( | |||||
"Options --outdated and --uptodate cannot be combined.") | |||||
packages = get_installed_distributions( | |||||
local_only=options.local, | |||||
user_only=options.user, | |||||
editables_only=options.editable, | |||||
include_editables=options.include_editable, | |||||
) | |||||
if options.outdated: | |||||
packages = self.get_outdated(packages, options) | |||||
elif options.uptodate: | |||||
packages = self.get_uptodate(packages, options) | |||||
if options.not_required: | |||||
packages = self.get_not_required(packages, options) | |||||
self.output_package_listing(packages, options) | |||||
def get_outdated(self, packages, options): | |||||
return [ | |||||
dist for dist in self.iter_packages_latest_infos(packages, options) | |||||
if dist.latest_version > dist.parsed_version | |||||
] | |||||
def get_uptodate(self, packages, options): | |||||
return [ | |||||
dist for dist in self.iter_packages_latest_infos(packages, options) | |||||
if dist.latest_version == dist.parsed_version | |||||
] | |||||
def get_not_required(self, packages, options): | |||||
dep_keys = set() | |||||
for dist in packages: | |||||
dep_keys.update(requirement.key for requirement in dist.requires()) | |||||
return {pkg for pkg in packages if pkg.key not in dep_keys} | |||||
def iter_packages_latest_infos(self, packages, options): | |||||
index_urls = [options.index_url] + options.extra_index_urls | |||||
if options.no_index: | |||||
logger.debug('Ignoring indexes: %s', ','.join(index_urls)) | |||||
index_urls = [] | |||||
dependency_links = [] | |||||
for dist in packages: | |||||
if dist.has_metadata('dependency_links.txt'): | |||||
dependency_links.extend( | |||||
dist.get_metadata_lines('dependency_links.txt'), | |||||
) | |||||
with self._build_session(options) as session: | |||||
finder = self._build_package_finder(options, index_urls, session) | |||||
finder.add_dependency_links(dependency_links) | |||||
for dist in packages: | |||||
typ = 'unknown' | |||||
all_candidates = finder.find_all_candidates(dist.key) | |||||
if not options.pre: | |||||
# Remove prereleases | |||||
all_candidates = [candidate for candidate in all_candidates | |||||
if not candidate.version.is_prerelease] | |||||
if not all_candidates: | |||||
continue | |||||
best_candidate = max(all_candidates, | |||||
key=finder._candidate_sort_key) | |||||
remote_version = best_candidate.version | |||||
if best_candidate.location.is_wheel: | |||||
typ = 'wheel' | |||||
else: | |||||
typ = 'sdist' | |||||
# This is dirty but makes the rest of the code much cleaner | |||||
dist.latest_version = remote_version | |||||
dist.latest_filetype = typ | |||||
yield dist | |||||
def output_legacy(self, dist, options): | |||||
if options.verbose >= 1: | |||||
return '%s (%s, %s, %s)' % ( | |||||
dist.project_name, | |||||
dist.version, | |||||
dist.location, | |||||
get_installer(dist), | |||||
) | |||||
elif dist_is_editable(dist): | |||||
return '%s (%s, %s)' % ( | |||||
dist.project_name, | |||||
dist.version, | |||||
dist.location, | |||||
) | |||||
else: | |||||
return '%s (%s)' % (dist.project_name, dist.version) | |||||
def output_legacy_latest(self, dist, options): | |||||
return '%s - Latest: %s [%s]' % ( | |||||
self.output_legacy(dist, options), | |||||
dist.latest_version, | |||||
dist.latest_filetype, | |||||
) | |||||
def output_package_listing(self, packages, options): | |||||
packages = sorted( | |||||
packages, | |||||
key=lambda dist: dist.project_name.lower(), | |||||
) | |||||
if options.list_format == 'columns' and packages: | |||||
data, header = format_for_columns(packages, options) | |||||
self.output_package_listing_columns(data, header) | |||||
elif options.list_format == 'freeze': | |||||
for dist in packages: | |||||
if options.verbose >= 1: | |||||
logger.info("%s==%s (%s)", dist.project_name, | |||||
dist.version, dist.location) | |||||
else: | |||||
logger.info("%s==%s", dist.project_name, dist.version) | |||||
elif options.list_format == 'json': | |||||
logger.info(format_for_json(packages, options)) | |||||
elif options.list_format == "legacy": | |||||
for dist in packages: | |||||
if options.outdated: | |||||
logger.info(self.output_legacy_latest(dist, options)) | |||||
else: | |||||
logger.info(self.output_legacy(dist, options)) | |||||
def output_package_listing_columns(self, data, header): | |||||
# insert the header first: we need to know the size of column names | |||||
if len(data) > 0: | |||||
data.insert(0, header) | |||||
pkg_strings, sizes = tabulate(data) | |||||
# Create and add a separator. | |||||
if len(data) > 0: | |||||
pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes))) | |||||
for val in pkg_strings: | |||||
logger.info(val) | |||||
def tabulate(vals): | |||||
# From pfmoore on GitHub: | |||||
# https://github.com/pypa/pip/issues/3651#issuecomment-216932564 | |||||
assert len(vals) > 0 | |||||
sizes = [0] * max(len(x) for x in vals) | |||||
for row in vals: | |||||
sizes = [max(s, len(str(c))) for s, c in zip_longest(sizes, row)] | |||||
result = [] | |||||
for row in vals: | |||||
display = " ".join([str(c).ljust(s) if c is not None else '' | |||||
for s, c in zip_longest(sizes, row)]) | |||||
result.append(display) | |||||
return result, sizes | |||||
def format_for_columns(pkgs, options): | |||||
""" | |||||
Convert the package data into something usable | |||||
by output_package_listing_columns. | |||||
""" | |||||
running_outdated = options.outdated | |||||
# Adjust the header for the `pip list --outdated` case. | |||||
if running_outdated: | |||||
header = ["Package", "Version", "Latest", "Type"] | |||||
else: | |||||
header = ["Package", "Version"] | |||||
data = [] | |||||
if options.verbose >= 1 or any(dist_is_editable(x) for x in pkgs): | |||||
header.append("Location") | |||||
if options.verbose >= 1: | |||||
header.append("Installer") | |||||
for proj in pkgs: | |||||
# if we're working on the 'outdated' list, separate out the | |||||
# latest_version and type | |||||
row = [proj.project_name, proj.version] | |||||
if running_outdated: | |||||
row.append(proj.latest_version) | |||||
row.append(proj.latest_filetype) | |||||
if options.verbose >= 1 or dist_is_editable(proj): | |||||
row.append(proj.location) | |||||
if options.verbose >= 1: | |||||
row.append(get_installer(proj)) | |||||
data.append(row) | |||||
return data, header | |||||
def format_for_json(packages, options): | |||||
data = [] | |||||
for dist in packages: | |||||
info = { | |||||
'name': dist.project_name, | |||||
'version': six.text_type(dist.version), | |||||
} | |||||
if options.verbose >= 1: | |||||
info['location'] = dist.location | |||||
info['installer'] = get_installer(dist) | |||||
if options.outdated: | |||||
info['latest_version'] = six.text_type(dist.latest_version) | |||||
info['latest_filetype'] = dist.latest_filetype | |||||
data.append(info) | |||||
return json.dumps(data) |
from __future__ import absolute_import | |||||
import logging | |||||
import sys | |||||
import textwrap | |||||
from collections import OrderedDict | |||||
from pip._vendor import pkg_resources | |||||
from pip._vendor.packaging.version import parse as parse_version | |||||
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is | |||||
# why we ignore the type on this import | |||||
from pip._vendor.six.moves import xmlrpc_client # type: ignore | |||||
from pip._internal.basecommand import SUCCESS, Command | |||||
from pip._internal.compat import get_terminal_size | |||||
from pip._internal.download import PipXmlrpcTransport | |||||
from pip._internal.exceptions import CommandError | |||||
from pip._internal.models import PyPI | |||||
from pip._internal.status_codes import NO_MATCHES_FOUND | |||||
from pip._internal.utils.logging import indent_log | |||||
logger = logging.getLogger(__name__) | |||||
class SearchCommand(Command): | |||||
"""Search for PyPI packages whose name or summary contains <query>.""" | |||||
name = 'search' | |||||
usage = """ | |||||
%prog [options] <query>""" | |||||
summary = 'Search PyPI for packages.' | |||||
ignore_require_venv = True | |||||
def __init__(self, *args, **kw): | |||||
super(SearchCommand, self).__init__(*args, **kw) | |||||
self.cmd_opts.add_option( | |||||
'-i', '--index', | |||||
dest='index', | |||||
metavar='URL', | |||||
default=PyPI.pypi_url, | |||||
help='Base URL of Python Package Index (default %default)') | |||||
self.parser.insert_option_group(0, self.cmd_opts) | |||||
def run(self, options, args): | |||||
if not args: | |||||
raise CommandError('Missing required argument (search query).') | |||||
query = args | |||||
pypi_hits = self.search(query, options) | |||||
hits = transform_hits(pypi_hits) | |||||
terminal_width = None | |||||
if sys.stdout.isatty(): | |||||
terminal_width = get_terminal_size()[0] | |||||
print_results(hits, terminal_width=terminal_width) | |||||
if pypi_hits: | |||||
return SUCCESS | |||||
return NO_MATCHES_FOUND | |||||
def search(self, query, options): | |||||
index_url = options.index | |||||
with self._build_session(options) as session: | |||||
transport = PipXmlrpcTransport(index_url, session) | |||||
pypi = xmlrpc_client.ServerProxy(index_url, transport) | |||||
hits = pypi.search({'name': query, 'summary': query}, 'or') | |||||
return hits | |||||
def transform_hits(hits): | |||||
""" | |||||
The list from pypi is really a list of versions. We want a list of | |||||
packages with the list of versions stored inline. This converts the | |||||
list from pypi into one we can use. | |||||
""" | |||||
packages = OrderedDict() | |||||
for hit in hits: | |||||
name = hit['name'] | |||||
summary = hit['summary'] | |||||
version = hit['version'] | |||||
if name not in packages.keys(): | |||||
packages[name] = { | |||||
'name': name, | |||||
'summary': summary, | |||||
'versions': [version], | |||||
} | |||||
else: | |||||
packages[name]['versions'].append(version) | |||||
# if this is the highest version, replace summary and score | |||||
if version == highest_version(packages[name]['versions']): | |||||
packages[name]['summary'] = summary | |||||
return list(packages.values()) | |||||
def print_results(hits, name_column_width=None, terminal_width=None): | |||||
if not hits: | |||||
return | |||||
if name_column_width is None: | |||||
name_column_width = max([ | |||||
len(hit['name']) + len(highest_version(hit.get('versions', ['-']))) | |||||
for hit in hits | |||||
]) + 4 | |||||
installed_packages = [p.project_name for p in pkg_resources.working_set] | |||||
for hit in hits: | |||||
name = hit['name'] | |||||
summary = hit['summary'] or '' | |||||
latest = highest_version(hit.get('versions', ['-'])) | |||||
if terminal_width is not None: | |||||
target_width = terminal_width - name_column_width - 5 | |||||
if target_width > 10: | |||||
# wrap and indent summary to fit terminal | |||||
summary = textwrap.wrap(summary, target_width) | |||||
summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) | |||||
line = '%-*s - %s' % (name_column_width, | |||||
'%s (%s)' % (name, latest), summary) | |||||
try: | |||||
logger.info(line) | |||||
if name in installed_packages: | |||||
dist = pkg_resources.get_distribution(name) | |||||
with indent_log(): | |||||
if dist.version == latest: | |||||
logger.info('INSTALLED: %s (latest)', dist.version) | |||||
else: | |||||
logger.info('INSTALLED: %s', dist.version) | |||||
logger.info('LATEST: %s', latest) | |||||
except UnicodeEncodeError: | |||||
pass | |||||
def highest_version(versions): | |||||
return max(versions, key=parse_version) |
from __future__ import absolute_import | |||||
import logging | |||||
import os | |||||
from email.parser import FeedParser # type: ignore | |||||
from pip._vendor import pkg_resources | |||||
from pip._vendor.packaging.utils import canonicalize_name | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.status_codes import ERROR, SUCCESS | |||||
logger = logging.getLogger(__name__) | |||||
class ShowCommand(Command): | |||||
"""Show information about one or more installed packages.""" | |||||
name = 'show' | |||||
usage = """ | |||||
%prog [options] <package> ...""" | |||||
summary = 'Show information about installed packages.' | |||||
ignore_require_venv = True | |||||
def __init__(self, *args, **kw): | |||||
super(ShowCommand, self).__init__(*args, **kw) | |||||
self.cmd_opts.add_option( | |||||
'-f', '--files', | |||||
dest='files', | |||||
action='store_true', | |||||
default=False, | |||||
help='Show the full list of installed files for each package.') | |||||
self.parser.insert_option_group(0, self.cmd_opts) | |||||
def run(self, options, args): | |||||
if not args: | |||||
logger.warning('ERROR: Please provide a package name or names.') | |||||
return ERROR | |||||
query = args | |||||
results = search_packages_info(query) | |||||
if not print_results( | |||||
results, list_files=options.files, verbose=options.verbose): | |||||
return ERROR | |||||
return SUCCESS | |||||
def search_packages_info(query): | |||||
""" | |||||
Gather details from installed distributions. Print distribution name, | |||||
version, location, and installed files. Installed files requires a | |||||
pip generated 'installed-files.txt' in the distributions '.egg-info' | |||||
directory. | |||||
""" | |||||
installed = {} | |||||
for p in pkg_resources.working_set: | |||||
installed[canonicalize_name(p.project_name)] = p | |||||
query_names = [canonicalize_name(name) for name in query] | |||||
for dist in [installed[pkg] for pkg in query_names if pkg in installed]: | |||||
package = { | |||||
'name': dist.project_name, | |||||
'version': dist.version, | |||||
'location': dist.location, | |||||
'requires': [dep.project_name for dep in dist.requires()], | |||||
} | |||||
file_list = None | |||||
metadata = None | |||||
if isinstance(dist, pkg_resources.DistInfoDistribution): | |||||
# RECORDs should be part of .dist-info metadatas | |||||
if dist.has_metadata('RECORD'): | |||||
lines = dist.get_metadata_lines('RECORD') | |||||
paths = [l.split(',')[0] for l in lines] | |||||
paths = [os.path.join(dist.location, p) for p in paths] | |||||
file_list = [os.path.relpath(p, dist.location) for p in paths] | |||||
if dist.has_metadata('METADATA'): | |||||
metadata = dist.get_metadata('METADATA') | |||||
else: | |||||
# Otherwise use pip's log for .egg-info's | |||||
if dist.has_metadata('installed-files.txt'): | |||||
paths = dist.get_metadata_lines('installed-files.txt') | |||||
paths = [os.path.join(dist.egg_info, p) for p in paths] | |||||
file_list = [os.path.relpath(p, dist.location) for p in paths] | |||||
if dist.has_metadata('PKG-INFO'): | |||||
metadata = dist.get_metadata('PKG-INFO') | |||||
if dist.has_metadata('entry_points.txt'): | |||||
entry_points = dist.get_metadata_lines('entry_points.txt') | |||||
package['entry_points'] = entry_points | |||||
if dist.has_metadata('INSTALLER'): | |||||
for line in dist.get_metadata_lines('INSTALLER'): | |||||
if line.strip(): | |||||
package['installer'] = line.strip() | |||||
break | |||||
# @todo: Should pkg_resources.Distribution have a | |||||
# `get_pkg_info` method? | |||||
feed_parser = FeedParser() | |||||
feed_parser.feed(metadata) | |||||
pkg_info_dict = feed_parser.close() | |||||
for key in ('metadata-version', 'summary', | |||||
'home-page', 'author', 'author-email', 'license'): | |||||
package[key] = pkg_info_dict.get(key) | |||||
# It looks like FeedParser cannot deal with repeated headers | |||||
classifiers = [] | |||||
for line in metadata.splitlines(): | |||||
if line.startswith('Classifier: '): | |||||
classifiers.append(line[len('Classifier: '):]) | |||||
package['classifiers'] = classifiers | |||||
if file_list: | |||||
package['files'] = sorted(file_list) | |||||
yield package | |||||
def print_results(distributions, list_files=False, verbose=False): | |||||
""" | |||||
Print the informations from installed distributions found. | |||||
""" | |||||
results_printed = False | |||||
for i, dist in enumerate(distributions): | |||||
results_printed = True | |||||
if i > 0: | |||||
logger.info("---") | |||||
name = dist.get('name', '') | |||||
required_by = [ | |||||
pkg.project_name for pkg in pkg_resources.working_set | |||||
if name in [required.name for required in pkg.requires()] | |||||
] | |||||
logger.info("Name: %s", name) | |||||
logger.info("Version: %s", dist.get('version', '')) | |||||
logger.info("Summary: %s", dist.get('summary', '')) | |||||
logger.info("Home-page: %s", dist.get('home-page', '')) | |||||
logger.info("Author: %s", dist.get('author', '')) | |||||
logger.info("Author-email: %s", dist.get('author-email', '')) | |||||
logger.info("License: %s", dist.get('license', '')) | |||||
logger.info("Location: %s", dist.get('location', '')) | |||||
logger.info("Requires: %s", ', '.join(dist.get('requires', []))) | |||||
logger.info("Required-by: %s", ', '.join(required_by)) | |||||
if verbose: | |||||
logger.info("Metadata-Version: %s", | |||||
dist.get('metadata-version', '')) | |||||
logger.info("Installer: %s", dist.get('installer', '')) | |||||
logger.info("Classifiers:") | |||||
for classifier in dist.get('classifiers', []): | |||||
logger.info(" %s", classifier) | |||||
logger.info("Entry-points:") | |||||
for entry in dist.get('entry_points', []): | |||||
logger.info(" %s", entry.strip()) | |||||
if list_files: | |||||
logger.info("Files:") | |||||
for line in dist.get('files', []): | |||||
logger.info(" %s", line.strip()) | |||||
if "files" not in dist: | |||||
logger.info("Cannot locate installed-files.txt") | |||||
return results_printed |
from __future__ import absolute_import | |||||
from pip._vendor.packaging.utils import canonicalize_name | |||||
from pip._internal.basecommand import Command | |||||
from pip._internal.exceptions import InstallationError | |||||
from pip._internal.req import InstallRequirement, parse_requirements | |||||
class UninstallCommand(Command): | |||||
""" | |||||
Uninstall packages. | |||||
pip is able to uninstall most installed packages. Known exceptions are: | |||||
- Pure distutils packages installed with ``python setup.py install``, which | |||||
leave behind no metadata to determine what files were installed. | |||||
- Script wrappers installed by ``python setup.py develop``. | |||||
""" | |||||
name = 'uninstall' | |||||
usage = """ | |||||
%prog [options] <package> ... | |||||
%prog [options] -r <requirements file> ...""" | |||||
summary = 'Uninstall packages.' | |||||
def __init__(self, *args, **kw): | |||||
super(UninstallCommand, self).__init__(*args, **kw) | |||||
self.cmd_opts.add_option( | |||||
'-r', '--requirement', | |||||
dest='requirements', | |||||
action='append', | |||||
default=[], | |||||
metavar='file', | |||||
help='Uninstall all the packages listed in the given requirements ' | |||||
'file. This option can be used multiple times.', | |||||
) | |||||
self.cmd_opts.add_option( | |||||
'-y', '--yes', | |||||
dest='yes', | |||||
action='store_true', | |||||
help="Don't ask for confirmation of uninstall deletions.") | |||||
self.parser.insert_option_group(0, self.cmd_opts) | |||||
def run(self, options, args): | |||||
with self._build_session(options) as session: | |||||
reqs_to_uninstall = {} | |||||
for name in args: | |||||
req = InstallRequirement.from_line( | |||||
name, isolated=options.isolated_mode, | |||||
) | |||||
if req.name: | |||||
reqs_to_uninstall[canonicalize_name(req.name)] = req | |||||
for filename in options.requirements: | |||||
for req in parse_requirements( | |||||
filename, | |||||
options=options, | |||||
session=session): | |||||
if req.name: | |||||
reqs_to_uninstall[canonicalize_name(req.name)] = req | |||||
if not reqs_to_uninstall: | |||||
raise InstallationError( | |||||
'You must give at least one requirement to %(name)s (see ' | |||||
'"pip help %(name)s")' % dict(name=self.name) | |||||
) | |||||
for req in reqs_to_uninstall.values(): | |||||
uninstall_pathset = req.uninstall( | |||||
auto_confirm=options.yes, verbose=self.verbosity > 0, | |||||
) | |||||
if uninstall_pathset: | |||||
uninstall_pathset.commit() |
# -*- coding: utf-8 -*- | |||||
from __future__ import absolute_import | |||||
import logging | |||||
import os | |||||
from pip._internal import cmdoptions | |||||
from pip._internal.basecommand import RequirementCommand | |||||
from pip._internal.cache import WheelCache | |||||
from pip._internal.exceptions import CommandError, PreviousBuildDirError | |||||
from pip._internal.operations.prepare import RequirementPreparer | |||||
from pip._internal.req import RequirementSet | |||||
from pip._internal.resolve import Resolver | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.wheel import WheelBuilder | |||||
logger = logging.getLogger(__name__) | |||||
class WheelCommand(RequirementCommand): | |||||
""" | |||||
Build Wheel archives for your requirements and dependencies. | |||||
Wheel is a built-package format, and offers the advantage of not | |||||
recompiling your software during every install. For more details, see the | |||||
wheel docs: https://wheel.readthedocs.io/en/latest/ | |||||
Requirements: setuptools>=0.8, and wheel. | |||||
'pip wheel' uses the bdist_wheel setuptools extension from the wheel | |||||
package to build individual wheels. | |||||
""" | |||||
name = 'wheel' | |||||
usage = """ | |||||
%prog [options] <requirement specifier> ... | |||||
%prog [options] -r <requirements file> ... | |||||
%prog [options] [-e] <vcs project url> ... | |||||
%prog [options] [-e] <local project path> ... | |||||
%prog [options] <archive url/path> ...""" | |||||
summary = 'Build wheels from your requirements.' | |||||
def __init__(self, *args, **kw): | |||||
super(WheelCommand, self).__init__(*args, **kw) | |||||
cmd_opts = self.cmd_opts | |||||
cmd_opts.add_option( | |||||
'-w', '--wheel-dir', | |||||
dest='wheel_dir', | |||||
metavar='dir', | |||||
default=os.curdir, | |||||
help=("Build wheels into <dir>, where the default is the " | |||||
"current working directory."), | |||||
) | |||||
cmd_opts.add_option(cmdoptions.no_binary()) | |||||
cmd_opts.add_option(cmdoptions.only_binary()) | |||||
cmd_opts.add_option( | |||||
'--build-option', | |||||
dest='build_options', | |||||
metavar='options', | |||||
action='append', | |||||
help="Extra arguments to be supplied to 'setup.py bdist_wheel'.", | |||||
) | |||||
cmd_opts.add_option(cmdoptions.no_build_isolation()) | |||||
cmd_opts.add_option(cmdoptions.constraints()) | |||||
cmd_opts.add_option(cmdoptions.editable()) | |||||
cmd_opts.add_option(cmdoptions.requirements()) | |||||
cmd_opts.add_option(cmdoptions.src()) | |||||
cmd_opts.add_option(cmdoptions.ignore_requires_python()) | |||||
cmd_opts.add_option(cmdoptions.no_deps()) | |||||
cmd_opts.add_option(cmdoptions.build_dir()) | |||||
cmd_opts.add_option(cmdoptions.progress_bar()) | |||||
cmd_opts.add_option( | |||||
'--global-option', | |||||
dest='global_options', | |||||
action='append', | |||||
metavar='options', | |||||
help="Extra global options to be supplied to the setup.py " | |||||
"call before the 'bdist_wheel' command.") | |||||
cmd_opts.add_option( | |||||
'--pre', | |||||
action='store_true', | |||||
default=False, | |||||
help=("Include pre-release and development versions. By default, " | |||||
"pip only finds stable versions."), | |||||
) | |||||
cmd_opts.add_option(cmdoptions.no_clean()) | |||||
cmd_opts.add_option(cmdoptions.require_hashes()) | |||||
index_opts = cmdoptions.make_option_group( | |||||
cmdoptions.index_group, | |||||
self.parser, | |||||
) | |||||
self.parser.insert_option_group(0, index_opts) | |||||
self.parser.insert_option_group(0, cmd_opts) | |||||
def run(self, options, args): | |||||
cmdoptions.check_install_build_global(options) | |||||
index_urls = [options.index_url] + options.extra_index_urls | |||||
if options.no_index: | |||||
logger.debug('Ignoring indexes: %s', ','.join(index_urls)) | |||||
index_urls = [] | |||||
if options.build_dir: | |||||
options.build_dir = os.path.abspath(options.build_dir) | |||||
options.src_dir = os.path.abspath(options.src_dir) | |||||
with self._build_session(options) as session: | |||||
finder = self._build_package_finder(options, session) | |||||
build_delete = (not (options.no_clean or options.build_dir)) | |||||
wheel_cache = WheelCache(options.cache_dir, options.format_control) | |||||
with TempDirectory( | |||||
options.build_dir, delete=build_delete, kind="wheel" | |||||
) as directory: | |||||
requirement_set = RequirementSet( | |||||
require_hashes=options.require_hashes, | |||||
) | |||||
try: | |||||
self.populate_requirement_set( | |||||
requirement_set, args, options, finder, session, | |||||
self.name, wheel_cache | |||||
) | |||||
preparer = RequirementPreparer( | |||||
build_dir=directory.path, | |||||
src_dir=options.src_dir, | |||||
download_dir=None, | |||||
wheel_download_dir=options.wheel_dir, | |||||
progress_bar=options.progress_bar, | |||||
build_isolation=options.build_isolation, | |||||
) | |||||
resolver = Resolver( | |||||
preparer=preparer, | |||||
finder=finder, | |||||
session=session, | |||||
wheel_cache=wheel_cache, | |||||
use_user_site=False, | |||||
upgrade_strategy="to-satisfy-only", | |||||
force_reinstall=False, | |||||
ignore_dependencies=options.ignore_dependencies, | |||||
ignore_requires_python=options.ignore_requires_python, | |||||
ignore_installed=True, | |||||
isolated=options.isolated_mode, | |||||
) | |||||
resolver.resolve(requirement_set) | |||||
# build wheels | |||||
wb = WheelBuilder( | |||||
finder, preparer, wheel_cache, | |||||
build_options=options.build_options or [], | |||||
global_options=options.global_options or [], | |||||
no_clean=options.no_clean, | |||||
) | |||||
wheels_built_successfully = wb.build( | |||||
requirement_set.requirements.values(), session=session, | |||||
) | |||||
if not wheels_built_successfully: | |||||
raise CommandError( | |||||
"Failed to build one or more wheels" | |||||
) | |||||
except PreviousBuildDirError: | |||||
options.no_clean = True | |||||
raise | |||||
finally: | |||||
if not options.no_clean: | |||||
requirement_set.cleanup_files() | |||||
wheel_cache.cleanup() |
"""Stuff that differs in different Python versions and platform | |||||
distributions.""" | |||||
from __future__ import absolute_import, division | |||||
import codecs | |||||
import locale | |||||
import logging | |||||
import os | |||||
import shutil | |||||
import sys | |||||
from pip._vendor.six import text_type | |||||
try: | |||||
import ipaddress | |||||
except ImportError: | |||||
try: | |||||
from pip._vendor import ipaddress # type: ignore | |||||
except ImportError: | |||||
import ipaddr as ipaddress # type: ignore | |||||
ipaddress.ip_address = ipaddress.IPAddress | |||||
ipaddress.ip_network = ipaddress.IPNetwork | |||||
__all__ = [ | |||||
"ipaddress", "uses_pycache", "console_to_str", "native_str", | |||||
"get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", | |||||
] | |||||
logger = logging.getLogger(__name__) | |||||
if sys.version_info >= (3, 4): | |||||
uses_pycache = True | |||||
from importlib.util import cache_from_source | |||||
else: | |||||
import imp | |||||
try: | |||||
cache_from_source = imp.cache_from_source # type: ignore | |||||
except AttributeError: | |||||
# does not use __pycache__ | |||||
cache_from_source = None | |||||
uses_pycache = cache_from_source is not None | |||||
if sys.version_info >= (3, 5): | |||||
backslashreplace_decode = "backslashreplace" | |||||
else: | |||||
# In version 3.4 and older, backslashreplace exists | |||||
# but does not support use for decoding. | |||||
# We implement our own replace handler for this | |||||
# situation, so that we can consistently use | |||||
# backslash replacement for all versions. | |||||
def backslashreplace_decode_fn(err): | |||||
raw_bytes = (err.object[i] for i in range(err.start, err.end)) | |||||
if sys.version_info[0] == 2: | |||||
# Python 2 gave us characters - convert to numeric bytes | |||||
raw_bytes = (ord(b) for b in raw_bytes) | |||||
return u"".join(u"\\x%x" % c for c in raw_bytes), err.end | |||||
codecs.register_error( | |||||
"backslashreplace_decode", | |||||
backslashreplace_decode_fn, | |||||
) | |||||
backslashreplace_decode = "backslashreplace_decode" | |||||
def console_to_str(data): | |||||
"""Return a string, safe for output, of subprocess output. | |||||
We assume the data is in the locale preferred encoding. | |||||
If it won't decode properly, we warn the user but decode as | |||||
best we can. | |||||
We also ensure that the output can be safely written to | |||||
standard output without encoding errors. | |||||
""" | |||||
# First, get the encoding we assume. This is the preferred | |||||
# encoding for the locale, unless that is not found, or | |||||
# it is ASCII, in which case assume UTF-8 | |||||
encoding = locale.getpreferredencoding() | |||||
if (not encoding) or codecs.lookup(encoding).name == "ascii": | |||||
encoding = "utf-8" | |||||
# Now try to decode the data - if we fail, warn the user and | |||||
# decode with replacement. | |||||
try: | |||||
s = data.decode(encoding) | |||||
except UnicodeDecodeError: | |||||
logger.warning( | |||||
"Subprocess output does not appear to be encoded as %s", | |||||
encoding, | |||||
) | |||||
s = data.decode(encoding, errors=backslashreplace_decode) | |||||
# Make sure we can print the output, by encoding it to the output | |||||
# encoding with replacement of unencodable characters, and then | |||||
# decoding again. | |||||
# We use stderr's encoding because it's less likely to be | |||||
# redirected and if we don't find an encoding we skip this | |||||
# step (on the assumption that output is wrapped by something | |||||
# that won't fail). | |||||
# The double getattr is to deal with the possibility that we're | |||||
# being called in a situation where sys.__stderr__ doesn't exist, | |||||
# or doesn't have an encoding attribute. Neither of these cases | |||||
# should occur in normal pip use, but there's no harm in checking | |||||
# in case people use pip in (unsupported) unusual situations. | |||||
output_encoding = getattr(getattr(sys, "__stderr__", None), | |||||
"encoding", None) | |||||
if output_encoding: | |||||
s = s.encode(output_encoding, errors="backslashreplace") | |||||
s = s.decode(output_encoding) | |||||
return s | |||||
if sys.version_info >= (3,): | |||||
def native_str(s, replace=False): | |||||
if isinstance(s, bytes): | |||||
return s.decode('utf-8', 'replace' if replace else 'strict') | |||||
return s | |||||
else: | |||||
def native_str(s, replace=False): | |||||
# Replace is ignored -- unicode to UTF-8 can't fail | |||||
if isinstance(s, text_type): | |||||
return s.encode('utf-8') | |||||
return s | |||||
def get_path_uid(path): | |||||
""" | |||||
Return path's uid. | |||||
Does not follow symlinks: | |||||
https://github.com/pypa/pip/pull/935#discussion_r5307003 | |||||
Placed this function in compat due to differences on AIX and | |||||
Jython, that should eventually go away. | |||||
:raises OSError: When path is a symlink or can't be read. | |||||
""" | |||||
if hasattr(os, 'O_NOFOLLOW'): | |||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) | |||||
file_uid = os.fstat(fd).st_uid | |||||
os.close(fd) | |||||
else: # AIX and Jython | |||||
# WARNING: time of check vulnerability, but best we can do w/o NOFOLLOW | |||||
if not os.path.islink(path): | |||||
# older versions of Jython don't have `os.fstat` | |||||
file_uid = os.stat(path).st_uid | |||||
else: | |||||
# raise OSError for parity with os.O_NOFOLLOW above | |||||
raise OSError( | |||||
"%s is a symlink; Will not return uid for symlinks" % path | |||||
) | |||||
return file_uid | |||||
def expanduser(path): | |||||
""" | |||||
Expand ~ and ~user constructions. | |||||
Includes a workaround for http://bugs.python.org/issue14768 | |||||
""" | |||||
expanded = os.path.expanduser(path) | |||||
if path.startswith('~/') and expanded.startswith('//'): | |||||
expanded = expanded[1:] | |||||
return expanded | |||||
# packages in the stdlib that may have installation metadata, but should not be | |||||
# considered 'installed'. this theoretically could be determined based on | |||||
# dist.location (py27:`sysconfig.get_paths()['stdlib']`, | |||||
# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may | |||||
# make this ineffective, so hard-coding | |||||
stdlib_pkgs = {"python", "wsgiref", "argparse"} | |||||
# windows detection, covers cpython and ironpython | |||||
WINDOWS = (sys.platform.startswith("win") or | |||||
(sys.platform == 'cli' and os.name == 'nt')) | |||||
def samefile(file1, file2): | |||||
"""Provide an alternative for os.path.samefile on Windows/Python2""" | |||||
if hasattr(os.path, 'samefile'): | |||||
return os.path.samefile(file1, file2) | |||||
else: | |||||
path1 = os.path.normcase(os.path.abspath(file1)) | |||||
path2 = os.path.normcase(os.path.abspath(file2)) | |||||
return path1 == path2 | |||||
if hasattr(shutil, 'get_terminal_size'): | |||||
def get_terminal_size(): | |||||
""" | |||||
Returns a tuple (x, y) representing the width(x) and the height(y) | |||||
in characters of the terminal window. | |||||
""" | |||||
return tuple(shutil.get_terminal_size()) | |||||
else: | |||||
def get_terminal_size(): | |||||
""" | |||||
Returns a tuple (x, y) representing the width(x) and the height(y) | |||||
in characters of the terminal window. | |||||
""" | |||||
def ioctl_GWINSZ(fd): | |||||
try: | |||||
import fcntl | |||||
import termios | |||||
import struct | |||||
cr = struct.unpack_from( | |||||
'hh', | |||||
fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') | |||||
) | |||||
except: | |||||
return None | |||||
if cr == (0, 0): | |||||
return None | |||||
return cr | |||||
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) | |||||
if not cr: | |||||
try: | |||||
fd = os.open(os.ctermid(), os.O_RDONLY) | |||||
cr = ioctl_GWINSZ(fd) | |||||
os.close(fd) | |||||
except: | |||||
pass | |||||
if not cr: | |||||
cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) | |||||
return int(cr[1]), int(cr[0]) |
"""Configuration management setup | |||||
Some terminology: | |||||
- name | |||||
As written in config files. | |||||
- value | |||||
Value associated with a name | |||||
- key | |||||
Name combined with it's section (section.name) | |||||
- variant | |||||
A single word describing where the configuration key-value pair came from | |||||
""" | |||||
import locale | |||||
import logging | |||||
import os | |||||
from pip._vendor import six | |||||
from pip._vendor.six.moves import configparser | |||||
from pip._internal.exceptions import ConfigurationError | |||||
from pip._internal.locations import ( | |||||
legacy_config_file, new_config_file, running_under_virtualenv, | |||||
site_config_files, venv_config_file, | |||||
) | |||||
from pip._internal.utils.misc import ensure_dir, enum | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple | |||||
RawConfigParser = configparser.RawConfigParser # Shorthand | |||||
Kind = NewType("Kind", str) | |||||
logger = logging.getLogger(__name__) | |||||
# NOTE: Maybe use the optionx attribute to normalize keynames. | |||||
def _normalize_name(name): | |||||
# type: (str) -> str | |||||
"""Make a name consistent regardless of source (environment or file) | |||||
""" | |||||
name = name.lower().replace('_', '-') | |||||
if name.startswith('--'): | |||||
name = name[2:] # only prefer long opts | |||||
return name | |||||
def _disassemble_key(name): | |||||
# type: (str) -> List[str] | |||||
return name.split(".", 1) | |||||
# The kinds of configurations there are. | |||||
kinds = enum( | |||||
USER="user", # User Specific | |||||
GLOBAL="global", # System Wide | |||||
VENV="venv", # Virtual Environment Specific | |||||
ENV="env", # from PIP_CONFIG_FILE | |||||
ENV_VAR="env-var", # from Environment Variables | |||||
) | |||||
class Configuration(object): | |||||
"""Handles management of configuration. | |||||
Provides an interface to accessing and managing configuration files. | |||||
This class converts provides an API that takes "section.key-name" style | |||||
keys and stores the value associated with it as "key-name" under the | |||||
section "section". | |||||
This allows for a clean interface wherein the both the section and the | |||||
key-name are preserved in an easy to manage form in the configuration files | |||||
and the data stored is also nice. | |||||
""" | |||||
def __init__(self, isolated, load_only=None): | |||||
# type: (bool, Kind) -> None | |||||
super(Configuration, self).__init__() | |||||
_valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.VENV, None] | |||||
if load_only not in _valid_load_only: | |||||
raise ConfigurationError( | |||||
"Got invalid value for load_only - should be one of {}".format( | |||||
", ".join(map(repr, _valid_load_only[:-1])) | |||||
) | |||||
) | |||||
self.isolated = isolated # type: bool | |||||
self.load_only = load_only # type: Optional[Kind] | |||||
# The order here determines the override order. | |||||
self._override_order = [ | |||||
kinds.GLOBAL, kinds.USER, kinds.VENV, kinds.ENV, kinds.ENV_VAR | |||||
] | |||||
self._ignore_env_names = ["version", "help"] | |||||
# Because we keep track of where we got the data from | |||||
self._parsers = { | |||||
variant: [] for variant in self._override_order | |||||
} # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] | |||||
self._config = { | |||||
variant: {} for variant in self._override_order | |||||
} # type: Dict[Kind, Dict[str, Any]] | |||||
self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] | |||||
def load(self): | |||||
# type: () -> None | |||||
"""Loads configuration from configuration files and environment | |||||
""" | |||||
self._load_config_files() | |||||
if not self.isolated: | |||||
self._load_environment_vars() | |||||
def get_file_to_edit(self): | |||||
# type: () -> Optional[str] | |||||
"""Returns the file with highest priority in configuration | |||||
""" | |||||
assert self.load_only is not None, \ | |||||
"Need to be specified a file to be editing" | |||||
try: | |||||
return self._get_parser_to_modify()[0] | |||||
except IndexError: | |||||
return None | |||||
def items(self): | |||||
# type: () -> Iterable[Tuple[str, Any]] | |||||
"""Returns key-value pairs like dict.items() representing the loaded | |||||
configuration | |||||
""" | |||||
return self._dictionary.items() | |||||
def get_value(self, key): | |||||
# type: (str) -> Any | |||||
"""Get a value from the configuration. | |||||
""" | |||||
try: | |||||
return self._dictionary[key] | |||||
except KeyError: | |||||
raise ConfigurationError("No such key - {}".format(key)) | |||||
def set_value(self, key, value): | |||||
# type: (str, Any) -> None | |||||
"""Modify a value in the configuration. | |||||
""" | |||||
self._ensure_have_load_only() | |||||
fname, parser = self._get_parser_to_modify() | |||||
if parser is not None: | |||||
section, name = _disassemble_key(key) | |||||
# Modify the parser and the configuration | |||||
if not parser.has_section(section): | |||||
parser.add_section(section) | |||||
parser.set(section, name, value) | |||||
self._config[self.load_only][key] = value | |||||
self._mark_as_modified(fname, parser) | |||||
def unset_value(self, key): | |||||
# type: (str) -> None | |||||
"""Unset a value in the configuration. | |||||
""" | |||||
self._ensure_have_load_only() | |||||
if key not in self._config[self.load_only]: | |||||
raise ConfigurationError("No such key - {}".format(key)) | |||||
fname, parser = self._get_parser_to_modify() | |||||
if parser is not None: | |||||
section, name = _disassemble_key(key) | |||||
# Remove the key in the parser | |||||
modified_something = False | |||||
if parser.has_section(section): | |||||
# Returns whether the option was removed or not | |||||
modified_something = parser.remove_option(section, name) | |||||
if modified_something: | |||||
# name removed from parser, section may now be empty | |||||
section_iter = iter(parser.items(section)) | |||||
try: | |||||
val = six.next(section_iter) | |||||
except StopIteration: | |||||
val = None | |||||
if val is None: | |||||
parser.remove_section(section) | |||||
self._mark_as_modified(fname, parser) | |||||
else: | |||||
raise ConfigurationError( | |||||
"Fatal Internal error [id=1]. Please report as a bug." | |||||
) | |||||
del self._config[self.load_only][key] | |||||
def save(self): | |||||
# type: () -> None | |||||
"""Save the currentin-memory state. | |||||
""" | |||||
self._ensure_have_load_only() | |||||
for fname, parser in self._modified_parsers: | |||||
logger.info("Writing to %s", fname) | |||||
# Ensure directory exists. | |||||
ensure_dir(os.path.dirname(fname)) | |||||
with open(fname, "w") as f: | |||||
parser.write(f) # type: ignore | |||||
# | |||||
# Private routines | |||||
# | |||||
def _ensure_have_load_only(self): | |||||
# type: () -> None | |||||
if self.load_only is None: | |||||
raise ConfigurationError("Needed a specific file to be modifying.") | |||||
logger.debug("Will be working with %s variant only", self.load_only) | |||||
@property | |||||
def _dictionary(self): | |||||
# type: () -> Dict[str, Any] | |||||
"""A dictionary representing the loaded configuration. | |||||
""" | |||||
# NOTE: Dictionaries are not populated if not loaded. So, conditionals | |||||
# are not needed here. | |||||
retval = {} | |||||
for variant in self._override_order: | |||||
retval.update(self._config[variant]) | |||||
return retval | |||||
def _load_config_files(self): | |||||
# type: () -> None | |||||
"""Loads configuration from configuration files | |||||
""" | |||||
config_files = dict(self._iter_config_files()) | |||||
if config_files[kinds.ENV][0:1] == [os.devnull]: | |||||
logger.debug( | |||||
"Skipping loading configuration files due to " | |||||
"environment's PIP_CONFIG_FILE being os.devnull" | |||||
) | |||||
return | |||||
for variant, files in config_files.items(): | |||||
for fname in files: | |||||
# If there's specific variant set in `load_only`, load only | |||||
# that variant, not the others. | |||||
if self.load_only is not None and variant != self.load_only: | |||||
logger.debug( | |||||
"Skipping file '%s' (variant: %s)", fname, variant | |||||
) | |||||
continue | |||||
parser = self._load_file(variant, fname) | |||||
# Keeping track of the parsers used | |||||
self._parsers[variant].append((fname, parser)) | |||||
def _load_file(self, variant, fname): | |||||
# type: (Kind, str) -> RawConfigParser | |||||
logger.debug("For variant '%s', will try loading '%s'", variant, fname) | |||||
parser = self._construct_parser(fname) | |||||
for section in parser.sections(): | |||||
items = parser.items(section) | |||||
self._config[variant].update(self._normalized_keys(section, items)) | |||||
return parser | |||||
def _construct_parser(self, fname): | |||||
# type: (str) -> RawConfigParser | |||||
parser = configparser.RawConfigParser() | |||||
# If there is no such file, don't bother reading it but create the | |||||
# parser anyway, to hold the data. | |||||
# Doing this is useful when modifying and saving files, where we don't | |||||
# need to construct a parser. | |||||
if os.path.exists(fname): | |||||
try: | |||||
parser.read(fname) | |||||
except UnicodeDecodeError: | |||||
raise ConfigurationError(( | |||||
"ERROR: " | |||||
"Configuration file contains invalid %s characters.\n" | |||||
"Please fix your configuration, located at %s\n" | |||||
) % (locale.getpreferredencoding(False), fname)) | |||||
return parser | |||||
def _load_environment_vars(self): | |||||
# type: () -> None | |||||
"""Loads configuration from environment variables | |||||
""" | |||||
self._config[kinds.ENV_VAR].update( | |||||
self._normalized_keys(":env:", self._get_environ_vars()) | |||||
) | |||||
def _normalized_keys(self, section, items): | |||||
# type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any] | |||||
"""Normalizes items to construct a dictionary with normalized keys. | |||||
This routine is where the names become keys and are made the same | |||||
regardless of source - configuration files or environment. | |||||
""" | |||||
normalized = {} | |||||
for name, val in items: | |||||
key = section + "." + _normalize_name(name) | |||||
normalized[key] = val | |||||
return normalized | |||||
def _get_environ_vars(self): | |||||
# type: () -> Iterable[Tuple[str, str]] | |||||
"""Returns a generator with all environmental vars with prefix PIP_""" | |||||
for key, val in os.environ.items(): | |||||
should_be_yielded = ( | |||||
key.startswith("PIP_") and | |||||
key[4:].lower() not in self._ignore_env_names | |||||
) | |||||
if should_be_yielded: | |||||
yield key[4:].lower(), val | |||||
# XXX: This is patched in the tests. | |||||
def _iter_config_files(self): | |||||
# type: () -> Iterable[Tuple[Kind, List[str]]] | |||||
"""Yields variant and configuration files associated with it. | |||||
This should be treated like items of a dictionary. | |||||
""" | |||||
# SMELL: Move the conditions out of this function | |||||
# environment variables have the lowest priority | |||||
config_file = os.environ.get('PIP_CONFIG_FILE', None) | |||||
if config_file is not None: | |||||
yield kinds.ENV, [config_file] | |||||
else: | |||||
yield kinds.ENV, [] | |||||
# at the base we have any global configuration | |||||
yield kinds.GLOBAL, list(site_config_files) | |||||
# per-user configuration next | |||||
should_load_user_config = not self.isolated and not ( | |||||
config_file and os.path.exists(config_file) | |||||
) | |||||
if should_load_user_config: | |||||
# The legacy config file is overridden by the new config file | |||||
yield kinds.USER, [legacy_config_file, new_config_file] | |||||
# finally virtualenv configuration first trumping others | |||||
if running_under_virtualenv(): | |||||
yield kinds.VENV, [venv_config_file] | |||||
def _get_parser_to_modify(self): | |||||
# type: () -> Tuple[str, RawConfigParser] | |||||
# Determine which parser to modify | |||||
parsers = self._parsers[self.load_only] | |||||
if not parsers: | |||||
# This should not happen if everything works correctly. | |||||
raise ConfigurationError( | |||||
"Fatal Internal error [id=2]. Please report as a bug." | |||||
) | |||||
# Use the highest priority parser. | |||||
return parsers[-1] | |||||
# XXX: This is patched in the tests. | |||||
def _mark_as_modified(self, fname, parser): | |||||
# type: (str, RawConfigParser) -> None | |||||
file_parser_tuple = (fname, parser) | |||||
if file_parser_tuple not in self._modified_parsers: | |||||
self._modified_parsers.append(file_parser_tuple) |
from __future__ import absolute_import | |||||
import cgi | |||||
import email.utils | |||||
import getpass | |||||
import json | |||||
import logging | |||||
import mimetypes | |||||
import os | |||||
import platform | |||||
import re | |||||
import shutil | |||||
import sys | |||||
from pip._vendor import requests, six, urllib3 | |||||
from pip._vendor.cachecontrol import CacheControlAdapter | |||||
from pip._vendor.cachecontrol.caches import FileCache | |||||
from pip._vendor.lockfile import LockError | |||||
from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter | |||||
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth | |||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response | |||||
from pip._vendor.requests.structures import CaseInsensitiveDict | |||||
from pip._vendor.requests.utils import get_netrc_auth | |||||
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is | |||||
# why we ignore the type on this import | |||||
from pip._vendor.six.moves import xmlrpc_client # type: ignore | |||||
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
from pip._vendor.six.moves.urllib import request as urllib_request | |||||
from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote | |||||
from pip._vendor.urllib3.util import IS_PYOPENSSL | |||||
import pip | |||||
from pip._internal.compat import WINDOWS | |||||
from pip._internal.exceptions import HashMismatch, InstallationError | |||||
from pip._internal.locations import write_delete_marker_file | |||||
from pip._internal.models import PyPI | |||||
from pip._internal.utils.encoding import auto_decode | |||||
from pip._internal.utils.filesystem import check_path_owner | |||||
from pip._internal.utils.glibc import libc_ver | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.utils.misc import ( | |||||
ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume, | |||||
display_path, format_size, get_installed_version, rmtree, splitext, | |||||
unpack_file, | |||||
) | |||||
from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.utils.ui import DownloadProgressProvider | |||||
from pip._internal.vcs import vcs | |||||
try: | |||||
import ssl # noqa | |||||
except ImportError: | |||||
ssl = None | |||||
HAS_TLS = (ssl is not None) or IS_PYOPENSSL | |||||
__all__ = ['get_file_content', | |||||
'is_url', 'url_to_path', 'path_to_url', | |||||
'is_archive_file', 'unpack_vcs_link', | |||||
'unpack_file_url', 'is_vcs_url', 'is_file_url', | |||||
'unpack_http_url', 'unpack_url'] | |||||
logger = logging.getLogger(__name__) | |||||
def user_agent(): | |||||
""" | |||||
Return a string representing the user agent. | |||||
""" | |||||
data = { | |||||
"installer": {"name": "pip", "version": pip.__version__}, | |||||
"python": platform.python_version(), | |||||
"implementation": { | |||||
"name": platform.python_implementation(), | |||||
}, | |||||
} | |||||
if data["implementation"]["name"] == 'CPython': | |||||
data["implementation"]["version"] = platform.python_version() | |||||
elif data["implementation"]["name"] == 'PyPy': | |||||
if sys.pypy_version_info.releaselevel == 'final': | |||||
pypy_version_info = sys.pypy_version_info[:3] | |||||
else: | |||||
pypy_version_info = sys.pypy_version_info | |||||
data["implementation"]["version"] = ".".join( | |||||
[str(x) for x in pypy_version_info] | |||||
) | |||||
elif data["implementation"]["name"] == 'Jython': | |||||
# Complete Guess | |||||
data["implementation"]["version"] = platform.python_version() | |||||
elif data["implementation"]["name"] == 'IronPython': | |||||
# Complete Guess | |||||
data["implementation"]["version"] = platform.python_version() | |||||
if sys.platform.startswith("linux"): | |||||
from pip._vendor import distro | |||||
distro_infos = dict(filter( | |||||
lambda x: x[1], | |||||
zip(["name", "version", "id"], distro.linux_distribution()), | |||||
)) | |||||
libc = dict(filter( | |||||
lambda x: x[1], | |||||
zip(["lib", "version"], libc_ver()), | |||||
)) | |||||
if libc: | |||||
distro_infos["libc"] = libc | |||||
if distro_infos: | |||||
data["distro"] = distro_infos | |||||
if sys.platform.startswith("darwin") and platform.mac_ver()[0]: | |||||
data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} | |||||
if platform.system(): | |||||
data.setdefault("system", {})["name"] = platform.system() | |||||
if platform.release(): | |||||
data.setdefault("system", {})["release"] = platform.release() | |||||
if platform.machine(): | |||||
data["cpu"] = platform.machine() | |||||
if HAS_TLS: | |||||
data["openssl_version"] = ssl.OPENSSL_VERSION | |||||
setuptools_version = get_installed_version("setuptools") | |||||
if setuptools_version is not None: | |||||
data["setuptools_version"] = setuptools_version | |||||
return "{data[installer][name]}/{data[installer][version]} {json}".format( | |||||
data=data, | |||||
json=json.dumps(data, separators=(",", ":"), sort_keys=True), | |||||
) | |||||
class MultiDomainBasicAuth(AuthBase): | |||||
def __init__(self, prompting=True): | |||||
self.prompting = prompting | |||||
self.passwords = {} | |||||
def __call__(self, req): | |||||
parsed = urllib_parse.urlparse(req.url) | |||||
# Get the netloc without any embedded credentials | |||||
netloc = parsed.netloc.rsplit("@", 1)[-1] | |||||
# Set the url of the request to the url without any credentials | |||||
req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) | |||||
# Use any stored credentials that we have for this netloc | |||||
username, password = self.passwords.get(netloc, (None, None)) | |||||
# Extract credentials embedded in the url if we have none stored | |||||
if username is None: | |||||
username, password = self.parse_credentials(parsed.netloc) | |||||
# Get creds from netrc if we still don't have them | |||||
if username is None and password is None: | |||||
netrc_auth = get_netrc_auth(req.url) | |||||
username, password = netrc_auth if netrc_auth else (None, None) | |||||
if username or password: | |||||
# Store the username and password | |||||
self.passwords[netloc] = (username, password) | |||||
# Send the basic auth with this request | |||||
req = HTTPBasicAuth(username or "", password or "")(req) | |||||
# Attach a hook to handle 401 responses | |||||
req.register_hook("response", self.handle_401) | |||||
return req | |||||
def handle_401(self, resp, **kwargs): | |||||
# We only care about 401 responses, anything else we want to just | |||||
# pass through the actual response | |||||
if resp.status_code != 401: | |||||
return resp | |||||
# We are not able to prompt the user so simply return the response | |||||
if not self.prompting: | |||||
return resp | |||||
parsed = urllib_parse.urlparse(resp.url) | |||||
# Prompt the user for a new username and password | |||||
username = six.moves.input("User for %s: " % parsed.netloc) | |||||
password = getpass.getpass("Password: ") | |||||
# Store the new username and password to use for future requests | |||||
if username or password: | |||||
self.passwords[parsed.netloc] = (username, password) | |||||
# Consume content and release the original connection to allow our new | |||||
# request to reuse the same one. | |||||
resp.content | |||||
resp.raw.release_conn() | |||||
# Add our new username and password to the request | |||||
req = HTTPBasicAuth(username or "", password or "")(resp.request) | |||||
# Send our new request | |||||
new_resp = resp.connection.send(req, **kwargs) | |||||
new_resp.history.append(resp) | |||||
return new_resp | |||||
def parse_credentials(self, netloc): | |||||
if "@" in netloc: | |||||
userinfo = netloc.rsplit("@", 1)[0] | |||||
if ":" in userinfo: | |||||
user, pwd = userinfo.split(":", 1) | |||||
return (urllib_unquote(user), urllib_unquote(pwd)) | |||||
return urllib_unquote(userinfo), None | |||||
return None, None | |||||
class LocalFSAdapter(BaseAdapter): | |||||
def send(self, request, stream=None, timeout=None, verify=None, cert=None, | |||||
proxies=None): | |||||
pathname = url_to_path(request.url) | |||||
resp = Response() | |||||
resp.status_code = 200 | |||||
resp.url = request.url | |||||
try: | |||||
stats = os.stat(pathname) | |||||
except OSError as exc: | |||||
resp.status_code = 404 | |||||
resp.raw = exc | |||||
else: | |||||
modified = email.utils.formatdate(stats.st_mtime, usegmt=True) | |||||
content_type = mimetypes.guess_type(pathname)[0] or "text/plain" | |||||
resp.headers = CaseInsensitiveDict({ | |||||
"Content-Type": content_type, | |||||
"Content-Length": stats.st_size, | |||||
"Last-Modified": modified, | |||||
}) | |||||
resp.raw = open(pathname, "rb") | |||||
resp.close = resp.raw.close | |||||
return resp | |||||
def close(self): | |||||
pass | |||||
class SafeFileCache(FileCache): | |||||
""" | |||||
A file based cache which is safe to use even when the target directory may | |||||
not be accessible or writable. | |||||
""" | |||||
def __init__(self, *args, **kwargs): | |||||
super(SafeFileCache, self).__init__(*args, **kwargs) | |||||
# Check to ensure that the directory containing our cache directory | |||||
# is owned by the user current executing pip. If it does not exist | |||||
# we will check the parent directory until we find one that does exist. | |||||
# If it is not owned by the user executing pip then we will disable | |||||
# the cache and log a warning. | |||||
if not check_path_owner(self.directory): | |||||
logger.warning( | |||||
"The directory '%s' or its parent directory is not owned by " | |||||
"the current user and the cache has been disabled. Please " | |||||
"check the permissions and owner of that directory. If " | |||||
"executing pip with sudo, you may want sudo's -H flag.", | |||||
self.directory, | |||||
) | |||||
# Set our directory to None to disable the Cache | |||||
self.directory = None | |||||
def get(self, *args, **kwargs): | |||||
# If we don't have a directory, then the cache should be a no-op. | |||||
if self.directory is None: | |||||
return | |||||
try: | |||||
return super(SafeFileCache, self).get(*args, **kwargs) | |||||
except (LockError, OSError, IOError): | |||||
# We intentionally silence this error, if we can't access the cache | |||||
# then we can just skip caching and process the request as if | |||||
# caching wasn't enabled. | |||||
pass | |||||
def set(self, *args, **kwargs): | |||||
# If we don't have a directory, then the cache should be a no-op. | |||||
if self.directory is None: | |||||
return | |||||
try: | |||||
return super(SafeFileCache, self).set(*args, **kwargs) | |||||
except (LockError, OSError, IOError): | |||||
# We intentionally silence this error, if we can't access the cache | |||||
# then we can just skip caching and process the request as if | |||||
# caching wasn't enabled. | |||||
pass | |||||
def delete(self, *args, **kwargs): | |||||
# If we don't have a directory, then the cache should be a no-op. | |||||
if self.directory is None: | |||||
return | |||||
try: | |||||
return super(SafeFileCache, self).delete(*args, **kwargs) | |||||
except (LockError, OSError, IOError): | |||||
# We intentionally silence this error, if we can't access the cache | |||||
# then we can just skip caching and process the request as if | |||||
# caching wasn't enabled. | |||||
pass | |||||
class InsecureHTTPAdapter(HTTPAdapter): | |||||
def cert_verify(self, conn, url, verify, cert): | |||||
conn.cert_reqs = 'CERT_NONE' | |||||
conn.ca_certs = None | |||||
class PipSession(requests.Session): | |||||
timeout = None | |||||
def __init__(self, *args, **kwargs): | |||||
retries = kwargs.pop("retries", 0) | |||||
cache = kwargs.pop("cache", None) | |||||
insecure_hosts = kwargs.pop("insecure_hosts", []) | |||||
super(PipSession, self).__init__(*args, **kwargs) | |||||
# Attach our User Agent to the request | |||||
self.headers["User-Agent"] = user_agent() | |||||
# Attach our Authentication handler to the session | |||||
self.auth = MultiDomainBasicAuth() | |||||
# Create our urllib3.Retry instance which will allow us to customize | |||||
# how we handle retries. | |||||
retries = urllib3.Retry( | |||||
# Set the total number of retries that a particular request can | |||||
# have. | |||||
total=retries, | |||||
# A 503 error from PyPI typically means that the Fastly -> Origin | |||||
# connection got interrupted in some way. A 503 error in general | |||||
# is typically considered a transient error so we'll go ahead and | |||||
# retry it. | |||||
# A 500 may indicate transient error in Amazon S3 | |||||
# A 520 or 527 - may indicate transient error in CloudFlare | |||||
status_forcelist=[500, 503, 520, 527], | |||||
# Add a small amount of back off between failed requests in | |||||
# order to prevent hammering the service. | |||||
backoff_factor=0.25, | |||||
) | |||||
# We want to _only_ cache responses on securely fetched origins. We do | |||||
# this because we can't validate the response of an insecurely fetched | |||||
# origin, and we don't want someone to be able to poison the cache and | |||||
# require manual eviction from the cache to fix it. | |||||
if cache: | |||||
secure_adapter = CacheControlAdapter( | |||||
cache=SafeFileCache(cache, use_dir_lock=True), | |||||
max_retries=retries, | |||||
) | |||||
else: | |||||
secure_adapter = HTTPAdapter(max_retries=retries) | |||||
# Our Insecure HTTPAdapter disables HTTPS validation. It does not | |||||
# support caching (see above) so we'll use it for all http:// URLs as | |||||
# well as any https:// host that we've marked as ignoring TLS errors | |||||
# for. | |||||
insecure_adapter = InsecureHTTPAdapter(max_retries=retries) | |||||
self.mount("https://", secure_adapter) | |||||
self.mount("http://", insecure_adapter) | |||||
# Enable file:// urls | |||||
self.mount("file://", LocalFSAdapter()) | |||||
# We want to use a non-validating adapter for any requests which are | |||||
# deemed insecure. | |||||
for host in insecure_hosts: | |||||
self.mount("https://{}/".format(host), insecure_adapter) | |||||
def request(self, method, url, *args, **kwargs): | |||||
# Allow setting a default timeout on a session | |||||
kwargs.setdefault("timeout", self.timeout) | |||||
# Dispatch the actual request | |||||
return super(PipSession, self).request(method, url, *args, **kwargs) | |||||
def get_file_content(url, comes_from=None, session=None): | |||||
"""Gets the content of a file; it may be a filename, file: URL, or | |||||
http: URL. Returns (location, content). Content is unicode. | |||||
:param url: File path or url. | |||||
:param comes_from: Origin description of requirements. | |||||
:param session: Instance of pip.download.PipSession. | |||||
""" | |||||
if session is None: | |||||
raise TypeError( | |||||
"get_file_content() missing 1 required keyword argument: 'session'" | |||||
) | |||||
match = _scheme_re.search(url) | |||||
if match: | |||||
scheme = match.group(1).lower() | |||||
if (scheme == 'file' and comes_from and | |||||
comes_from.startswith('http')): | |||||
raise InstallationError( | |||||
'Requirements file %s references URL %s, which is local' | |||||
% (comes_from, url)) | |||||
if scheme == 'file': | |||||
path = url.split(':', 1)[1] | |||||
path = path.replace('\\', '/') | |||||
match = _url_slash_drive_re.match(path) | |||||
if match: | |||||
path = match.group(1) + ':' + path.split('|', 1)[1] | |||||
path = urllib_parse.unquote(path) | |||||
if path.startswith('/'): | |||||
path = '/' + path.lstrip('/') | |||||
url = path | |||||
else: | |||||
# FIXME: catch some errors | |||||
resp = session.get(url) | |||||
resp.raise_for_status() | |||||
return resp.url, resp.text | |||||
try: | |||||
with open(url, 'rb') as f: | |||||
content = auto_decode(f.read()) | |||||
except IOError as exc: | |||||
raise InstallationError( | |||||
'Could not open requirements file: %s' % str(exc) | |||||
) | |||||
return url, content | |||||
_scheme_re = re.compile(r'^(http|https|file):', re.I) | |||||
_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) | |||||
def is_url(name): | |||||
"""Returns true if the name looks like a URL""" | |||||
if ':' not in name: | |||||
return False | |||||
scheme = name.split(':', 1)[0].lower() | |||||
return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes | |||||
def url_to_path(url): | |||||
""" | |||||
Convert a file: URL to a path. | |||||
""" | |||||
assert url.startswith('file:'), ( | |||||
"You can only turn file: urls into filenames (not %r)" % url) | |||||
_, netloc, path, _, _ = urllib_parse.urlsplit(url) | |||||
# if we have a UNC path, prepend UNC share notation | |||||
if netloc: | |||||
netloc = '\\\\' + netloc | |||||
path = urllib_request.url2pathname(netloc + path) | |||||
return path | |||||
def path_to_url(path): | |||||
""" | |||||
Convert a path to a file: URL. The path will be made absolute and have | |||||
quoted path parts. | |||||
""" | |||||
path = os.path.normpath(os.path.abspath(path)) | |||||
url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) | |||||
return url | |||||
def is_archive_file(name): | |||||
"""Return True if `name` is a considered as an archive file.""" | |||||
ext = splitext(name)[1].lower() | |||||
if ext in ARCHIVE_EXTENSIONS: | |||||
return True | |||||
return False | |||||
def unpack_vcs_link(link, location): | |||||
vcs_backend = _get_used_vcs_backend(link) | |||||
vcs_backend.unpack(location) | |||||
def _get_used_vcs_backend(link): | |||||
for backend in vcs.backends: | |||||
if link.scheme in backend.schemes: | |||||
vcs_backend = backend(link.url) | |||||
return vcs_backend | |||||
def is_vcs_url(link): | |||||
return bool(_get_used_vcs_backend(link)) | |||||
def is_file_url(link): | |||||
return link.url.lower().startswith('file:') | |||||
def is_dir_url(link): | |||||
"""Return whether a file:// Link points to a directory. | |||||
``link`` must not have any other scheme but file://. Call is_file_url() | |||||
first. | |||||
""" | |||||
link_path = url_to_path(link.url_without_fragment) | |||||
return os.path.isdir(link_path) | |||||
def _progress_indicator(iterable, *args, **kwargs): | |||||
return iterable | |||||
def _download_url(resp, link, content_file, hashes, progress_bar): | |||||
try: | |||||
total_length = int(resp.headers['content-length']) | |||||
except (ValueError, KeyError, TypeError): | |||||
total_length = 0 | |||||
cached_resp = getattr(resp, "from_cache", False) | |||||
if logger.getEffectiveLevel() > logging.INFO: | |||||
show_progress = False | |||||
elif cached_resp: | |||||
show_progress = False | |||||
elif total_length > (40 * 1000): | |||||
show_progress = True | |||||
elif not total_length: | |||||
show_progress = True | |||||
else: | |||||
show_progress = False | |||||
show_url = link.show_url | |||||
def resp_read(chunk_size): | |||||
try: | |||||
# Special case for urllib3. | |||||
for chunk in resp.raw.stream( | |||||
chunk_size, | |||||
# We use decode_content=False here because we don't | |||||
# want urllib3 to mess with the raw bytes we get | |||||
# from the server. If we decompress inside of | |||||
# urllib3 then we cannot verify the checksum | |||||
# because the checksum will be of the compressed | |||||
# file. This breakage will only occur if the | |||||
# server adds a Content-Encoding header, which | |||||
# depends on how the server was configured: | |||||
# - Some servers will notice that the file isn't a | |||||
# compressible file and will leave the file alone | |||||
# and with an empty Content-Encoding | |||||
# - Some servers will notice that the file is | |||||
# already compressed and will leave the file | |||||
# alone and will add a Content-Encoding: gzip | |||||
# header | |||||
# - Some servers won't notice anything at all and | |||||
# will take a file that's already been compressed | |||||
# and compress it again and set the | |||||
# Content-Encoding: gzip header | |||||
# | |||||
# By setting this not to decode automatically we | |||||
# hope to eliminate problems with the second case. | |||||
decode_content=False): | |||||
yield chunk | |||||
except AttributeError: | |||||
# Standard file-like object. | |||||
while True: | |||||
chunk = resp.raw.read(chunk_size) | |||||
if not chunk: | |||||
break | |||||
yield chunk | |||||
def written_chunks(chunks): | |||||
for chunk in chunks: | |||||
content_file.write(chunk) | |||||
yield chunk | |||||
progress_indicator = _progress_indicator | |||||
if link.netloc == PyPI.netloc: | |||||
url = show_url | |||||
else: | |||||
url = link.url_without_fragment | |||||
if show_progress: # We don't show progress on cached responses | |||||
progress_indicator = DownloadProgressProvider(progress_bar, | |||||
max=total_length) | |||||
if total_length: | |||||
logger.info("Downloading %s (%s)", url, format_size(total_length)) | |||||
else: | |||||
logger.info("Downloading %s", url) | |||||
elif cached_resp: | |||||
logger.info("Using cached %s", url) | |||||
else: | |||||
logger.info("Downloading %s", url) | |||||
logger.debug('Downloading from URL %s', link) | |||||
downloaded_chunks = written_chunks( | |||||
progress_indicator( | |||||
resp_read(CONTENT_CHUNK_SIZE), | |||||
CONTENT_CHUNK_SIZE | |||||
) | |||||
) | |||||
if hashes: | |||||
hashes.check_against_chunks(downloaded_chunks) | |||||
else: | |||||
consume(downloaded_chunks) | |||||
def _copy_file(filename, location, link): | |||||
copy = True | |||||
download_location = os.path.join(location, link.filename) | |||||
if os.path.exists(download_location): | |||||
response = ask_path_exists( | |||||
'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' % | |||||
display_path(download_location), ('i', 'w', 'b', 'a')) | |||||
if response == 'i': | |||||
copy = False | |||||
elif response == 'w': | |||||
logger.warning('Deleting %s', display_path(download_location)) | |||||
os.remove(download_location) | |||||
elif response == 'b': | |||||
dest_file = backup_dir(download_location) | |||||
logger.warning( | |||||
'Backing up %s to %s', | |||||
display_path(download_location), | |||||
display_path(dest_file), | |||||
) | |||||
shutil.move(download_location, dest_file) | |||||
elif response == 'a': | |||||
sys.exit(-1) | |||||
if copy: | |||||
shutil.copy(filename, download_location) | |||||
logger.info('Saved %s', display_path(download_location)) | |||||
def unpack_http_url(link, location, download_dir=None, | |||||
session=None, hashes=None, progress_bar="on"): | |||||
if session is None: | |||||
raise TypeError( | |||||
"unpack_http_url() missing 1 required keyword argument: 'session'" | |||||
) | |||||
with TempDirectory(kind="unpack") as temp_dir: | |||||
# If a download dir is specified, is the file already downloaded there? | |||||
already_downloaded_path = None | |||||
if download_dir: | |||||
already_downloaded_path = _check_download_dir(link, | |||||
download_dir, | |||||
hashes) | |||||
if already_downloaded_path: | |||||
from_path = already_downloaded_path | |||||
content_type = mimetypes.guess_type(from_path)[0] | |||||
else: | |||||
# let's download to a tmp dir | |||||
from_path, content_type = _download_http_url(link, | |||||
session, | |||||
temp_dir.path, | |||||
hashes, | |||||
progress_bar) | |||||
# unpack the archive to the build dir location. even when only | |||||
# downloading archives, they have to be unpacked to parse dependencies | |||||
unpack_file(from_path, location, content_type, link) | |||||
# a download dir is specified; let's copy the archive there | |||||
if download_dir and not already_downloaded_path: | |||||
_copy_file(from_path, download_dir, link) | |||||
if not already_downloaded_path: | |||||
os.unlink(from_path) | |||||
def unpack_file_url(link, location, download_dir=None, hashes=None): | |||||
"""Unpack link into location. | |||||
If download_dir is provided and link points to a file, make a copy | |||||
of the link file inside download_dir. | |||||
""" | |||||
link_path = url_to_path(link.url_without_fragment) | |||||
# If it's a url to a local directory | |||||
if is_dir_url(link): | |||||
if os.path.isdir(location): | |||||
rmtree(location) | |||||
shutil.copytree(link_path, location, symlinks=True) | |||||
if download_dir: | |||||
logger.info('Link is a directory, ignoring download_dir') | |||||
return | |||||
# If --require-hashes is off, `hashes` is either empty, the | |||||
# link's embedded hash, or MissingHashes; it is required to | |||||
# match. If --require-hashes is on, we are satisfied by any | |||||
# hash in `hashes` matching: a URL-based or an option-based | |||||
# one; no internet-sourced hash will be in `hashes`. | |||||
if hashes: | |||||
hashes.check_against_path(link_path) | |||||
# If a download dir is specified, is the file already there and valid? | |||||
already_downloaded_path = None | |||||
if download_dir: | |||||
already_downloaded_path = _check_download_dir(link, | |||||
download_dir, | |||||
hashes) | |||||
if already_downloaded_path: | |||||
from_path = already_downloaded_path | |||||
else: | |||||
from_path = link_path | |||||
content_type = mimetypes.guess_type(from_path)[0] | |||||
# unpack the archive to the build dir location. even when only downloading | |||||
# archives, they have to be unpacked to parse dependencies | |||||
unpack_file(from_path, location, content_type, link) | |||||
# a download dir is specified and not already downloaded | |||||
if download_dir and not already_downloaded_path: | |||||
_copy_file(from_path, download_dir, link) | |||||
def _copy_dist_from_dir(link_path, location): | |||||
"""Copy distribution files in `link_path` to `location`. | |||||
Invoked when user requests to install a local directory. E.g.: | |||||
pip install . | |||||
pip install ~/dev/git-repos/python-prompt-toolkit | |||||
""" | |||||
# Note: This is currently VERY SLOW if you have a lot of data in the | |||||
# directory, because it copies everything with `shutil.copytree`. | |||||
# What it should really do is build an sdist and install that. | |||||
# See https://github.com/pypa/pip/issues/2195 | |||||
if os.path.isdir(location): | |||||
rmtree(location) | |||||
# build an sdist | |||||
setup_py = 'setup.py' | |||||
sdist_args = [sys.executable] | |||||
sdist_args.append('-c') | |||||
sdist_args.append(SETUPTOOLS_SHIM % setup_py) | |||||
sdist_args.append('sdist') | |||||
sdist_args += ['--dist-dir', location] | |||||
logger.info('Running setup.py sdist for %s', link_path) | |||||
with indent_log(): | |||||
call_subprocess(sdist_args, cwd=link_path, show_stdout=False) | |||||
# unpack sdist into `location` | |||||
sdist = os.path.join(location, os.listdir(location)[0]) | |||||
logger.info('Unpacking sdist %s into %s', sdist, location) | |||||
unpack_file(sdist, location, content_type=None, link=None) | |||||
class PipXmlrpcTransport(xmlrpc_client.Transport): | |||||
"""Provide a `xmlrpclib.Transport` implementation via a `PipSession` | |||||
object. | |||||
""" | |||||
def __init__(self, index_url, session, use_datetime=False): | |||||
xmlrpc_client.Transport.__init__(self, use_datetime) | |||||
index_parts = urllib_parse.urlparse(index_url) | |||||
self._scheme = index_parts.scheme | |||||
self._session = session | |||||
def request(self, host, handler, request_body, verbose=False): | |||||
parts = (self._scheme, host, handler, None, None, None) | |||||
url = urllib_parse.urlunparse(parts) | |||||
try: | |||||
headers = {'Content-Type': 'text/xml'} | |||||
response = self._session.post(url, data=request_body, | |||||
headers=headers, stream=True) | |||||
response.raise_for_status() | |||||
self.verbose = verbose | |||||
return self.parse_response(response.raw) | |||||
except requests.HTTPError as exc: | |||||
logger.critical( | |||||
"HTTP error %s while getting %s", | |||||
exc.response.status_code, url, | |||||
) | |||||
raise | |||||
def unpack_url(link, location, download_dir=None, | |||||
only_download=False, session=None, hashes=None, | |||||
progress_bar="on"): | |||||
"""Unpack link. | |||||
If link is a VCS link: | |||||
if only_download, export into download_dir and ignore location | |||||
else unpack into location | |||||
for other types of link: | |||||
- unpack into location | |||||
- if download_dir, copy the file into download_dir | |||||
- if only_download, mark location for deletion | |||||
:param hashes: A Hashes object, one of whose embedded hashes must match, | |||||
or HashMismatch will be raised. If the Hashes is empty, no matches are | |||||
required, and unhashable types of requirements (like VCS ones, which | |||||
would ordinarily raise HashUnsupported) are allowed. | |||||
""" | |||||
# non-editable vcs urls | |||||
if is_vcs_url(link): | |||||
unpack_vcs_link(link, location) | |||||
# file urls | |||||
elif is_file_url(link): | |||||
unpack_file_url(link, location, download_dir, hashes=hashes) | |||||
# http urls | |||||
else: | |||||
if session is None: | |||||
session = PipSession() | |||||
unpack_http_url( | |||||
link, | |||||
location, | |||||
download_dir, | |||||
session, | |||||
hashes=hashes, | |||||
progress_bar=progress_bar | |||||
) | |||||
if only_download: | |||||
write_delete_marker_file(location) | |||||
def _download_http_url(link, session, temp_dir, hashes, progress_bar): | |||||
"""Download link url into temp_dir using provided session""" | |||||
target_url = link.url.split('#', 1)[0] | |||||
try: | |||||
resp = session.get( | |||||
target_url, | |||||
# We use Accept-Encoding: identity here because requests | |||||
# defaults to accepting compressed responses. This breaks in | |||||
# a variety of ways depending on how the server is configured. | |||||
# - Some servers will notice that the file isn't a compressible | |||||
# file and will leave the file alone and with an empty | |||||
# Content-Encoding | |||||
# - Some servers will notice that the file is already | |||||
# compressed and will leave the file alone and will add a | |||||
# Content-Encoding: gzip header | |||||
# - Some servers won't notice anything at all and will take | |||||
# a file that's already been compressed and compress it again | |||||
# and set the Content-Encoding: gzip header | |||||
# By setting this to request only the identity encoding We're | |||||
# hoping to eliminate the third case. Hopefully there does not | |||||
# exist a server which when given a file will notice it is | |||||
# already compressed and that you're not asking for a | |||||
# compressed file and will then decompress it before sending | |||||
# because if that's the case I don't think it'll ever be | |||||
# possible to make this work. | |||||
headers={"Accept-Encoding": "identity"}, | |||||
stream=True, | |||||
) | |||||
resp.raise_for_status() | |||||
except requests.HTTPError as exc: | |||||
logger.critical( | |||||
"HTTP error %s while getting %s", exc.response.status_code, link, | |||||
) | |||||
raise | |||||
content_type = resp.headers.get('content-type', '') | |||||
filename = link.filename # fallback | |||||
# Have a look at the Content-Disposition header for a better guess | |||||
content_disposition = resp.headers.get('content-disposition') | |||||
if content_disposition: | |||||
type, params = cgi.parse_header(content_disposition) | |||||
# We use ``or`` here because we don't want to use an "empty" value | |||||
# from the filename param. | |||||
filename = params.get('filename') or filename | |||||
ext = splitext(filename)[1] | |||||
if not ext: | |||||
ext = mimetypes.guess_extension(content_type) | |||||
if ext: | |||||
filename += ext | |||||
if not ext and link.url != resp.url: | |||||
ext = os.path.splitext(resp.url)[1] | |||||
if ext: | |||||
filename += ext | |||||
file_path = os.path.join(temp_dir, filename) | |||||
with open(file_path, 'wb') as content_file: | |||||
_download_url(resp, link, content_file, hashes, progress_bar) | |||||
return file_path, content_type | |||||
def _check_download_dir(link, download_dir, hashes): | |||||
""" Check download_dir for previously downloaded file with correct hash | |||||
If a correct file is found return its path else None | |||||
""" | |||||
download_path = os.path.join(download_dir, link.filename) | |||||
if os.path.exists(download_path): | |||||
# If already downloaded, does its hash match? | |||||
logger.info('File was already downloaded %s', download_path) | |||||
if hashes: | |||||
try: | |||||
hashes.check_against_path(download_path) | |||||
except HashMismatch: | |||||
logger.warning( | |||||
'Previously-downloaded file %s has bad hash. ' | |||||
'Re-downloading.', | |||||
download_path | |||||
) | |||||
os.unlink(download_path) | |||||
return None | |||||
return download_path | |||||
return None |
"""Exceptions used throughout package""" | |||||
from __future__ import absolute_import | |||||
from itertools import chain, groupby, repeat | |||||
from pip._vendor.six import iteritems | |||||
class PipError(Exception): | |||||
"""Base pip exception""" | |||||
class ConfigurationError(PipError): | |||||
"""General exception in configuration""" | |||||
class InstallationError(PipError): | |||||
"""General exception during installation""" | |||||
class UninstallationError(PipError): | |||||
"""General exception during uninstallation""" | |||||
class DistributionNotFound(InstallationError): | |||||
"""Raised when a distribution cannot be found to satisfy a requirement""" | |||||
class RequirementsFileParseError(InstallationError): | |||||
"""Raised when a general error occurs parsing a requirements file line.""" | |||||
class BestVersionAlreadyInstalled(PipError): | |||||
"""Raised when the most up-to-date version of a package is already | |||||
installed.""" | |||||
class BadCommand(PipError): | |||||
"""Raised when virtualenv or a command is not found""" | |||||
class CommandError(PipError): | |||||
"""Raised when there is an error in command-line arguments""" | |||||
class PreviousBuildDirError(PipError): | |||||
"""Raised when there's a previous conflicting build directory""" | |||||
class InvalidWheelFilename(InstallationError): | |||||
"""Invalid wheel filename.""" | |||||
class UnsupportedWheel(InstallationError): | |||||
"""Unsupported wheel.""" | |||||
class HashErrors(InstallationError): | |||||
"""Multiple HashError instances rolled into one for reporting""" | |||||
def __init__(self): | |||||
self.errors = [] | |||||
def append(self, error): | |||||
self.errors.append(error) | |||||
def __str__(self): | |||||
lines = [] | |||||
self.errors.sort(key=lambda e: e.order) | |||||
for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__): | |||||
lines.append(cls.head) | |||||
lines.extend(e.body() for e in errors_of_cls) | |||||
if lines: | |||||
return '\n'.join(lines) | |||||
def __nonzero__(self): | |||||
return bool(self.errors) | |||||
def __bool__(self): | |||||
return self.__nonzero__() | |||||
class HashError(InstallationError): | |||||
""" | |||||
A failure to verify a package against known-good hashes | |||||
:cvar order: An int sorting hash exception classes by difficulty of | |||||
recovery (lower being harder), so the user doesn't bother fretting | |||||
about unpinned packages when he has deeper issues, like VCS | |||||
dependencies, to deal with. Also keeps error reports in a | |||||
deterministic order. | |||||
:cvar head: A section heading for display above potentially many | |||||
exceptions of this kind | |||||
:ivar req: The InstallRequirement that triggered this error. This is | |||||
pasted on after the exception is instantiated, because it's not | |||||
typically available earlier. | |||||
""" | |||||
req = None | |||||
head = '' | |||||
def body(self): | |||||
"""Return a summary of me for display under the heading. | |||||
This default implementation simply prints a description of the | |||||
triggering requirement. | |||||
:param req: The InstallRequirement that provoked this error, with | |||||
populate_link() having already been called | |||||
""" | |||||
return ' %s' % self._requirement_name() | |||||
def __str__(self): | |||||
return '%s\n%s' % (self.head, self.body()) | |||||
def _requirement_name(self): | |||||
"""Return a description of the requirement that triggered me. | |||||
This default implementation returns long description of the req, with | |||||
line numbers | |||||
""" | |||||
return str(self.req) if self.req else 'unknown package' | |||||
class VcsHashUnsupported(HashError): | |||||
"""A hash was provided for a version-control-system-based requirement, but | |||||
we don't have a method for hashing those.""" | |||||
order = 0 | |||||
head = ("Can't verify hashes for these requirements because we don't " | |||||
"have a way to hash version control repositories:") | |||||
class DirectoryUrlHashUnsupported(HashError): | |||||
"""A hash was provided for a version-control-system-based requirement, but | |||||
we don't have a method for hashing those.""" | |||||
order = 1 | |||||
head = ("Can't verify hashes for these file:// requirements because they " | |||||
"point to directories:") | |||||
class HashMissing(HashError): | |||||
"""A hash was needed for a requirement but is absent.""" | |||||
order = 2 | |||||
head = ('Hashes are required in --require-hashes mode, but they are ' | |||||
'missing from some requirements. Here is a list of those ' | |||||
'requirements along with the hashes their downloaded archives ' | |||||
'actually had. Add lines like these to your requirements files to ' | |||||
'prevent tampering. (If you did not enable --require-hashes ' | |||||
'manually, note that it turns on automatically when any package ' | |||||
'has a hash.)') | |||||
def __init__(self, gotten_hash): | |||||
""" | |||||
:param gotten_hash: The hash of the (possibly malicious) archive we | |||||
just downloaded | |||||
""" | |||||
self.gotten_hash = gotten_hash | |||||
def body(self): | |||||
# Dodge circular import. | |||||
from pip._internal.utils.hashes import FAVORITE_HASH | |||||
package = None | |||||
if self.req: | |||||
# In the case of URL-based requirements, display the original URL | |||||
# seen in the requirements file rather than the package name, | |||||
# so the output can be directly copied into the requirements file. | |||||
package = (self.req.original_link if self.req.original_link | |||||
# In case someone feeds something downright stupid | |||||
# to InstallRequirement's constructor. | |||||
else getattr(self.req, 'req', None)) | |||||
return ' %s --hash=%s:%s' % (package or 'unknown package', | |||||
FAVORITE_HASH, | |||||
self.gotten_hash) | |||||
class HashUnpinned(HashError): | |||||
"""A requirement had a hash specified but was not pinned to a specific | |||||
version.""" | |||||
order = 3 | |||||
head = ('In --require-hashes mode, all requirements must have their ' | |||||
'versions pinned with ==. These do not:') | |||||
class HashMismatch(HashError): | |||||
""" | |||||
Distribution file hash values don't match. | |||||
:ivar package_name: The name of the package that triggered the hash | |||||
mismatch. Feel free to write to this after the exception is raise to | |||||
improve its error message. | |||||
""" | |||||
order = 4 | |||||
head = ('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS ' | |||||
'FILE. If you have updated the package versions, please update ' | |||||
'the hashes. Otherwise, examine the package contents carefully; ' | |||||
'someone may have tampered with them.') | |||||
def __init__(self, allowed, gots): | |||||
""" | |||||
:param allowed: A dict of algorithm names pointing to lists of allowed | |||||
hex digests | |||||
:param gots: A dict of algorithm names pointing to hashes we | |||||
actually got from the files under suspicion | |||||
""" | |||||
self.allowed = allowed | |||||
self.gots = gots | |||||
def body(self): | |||||
return ' %s:\n%s' % (self._requirement_name(), | |||||
self._hash_comparison()) | |||||
def _hash_comparison(self): | |||||
""" | |||||
Return a comparison of actual and expected hash values. | |||||
Example:: | |||||
Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde | |||||
or 123451234512345123451234512345123451234512345 | |||||
Got bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef | |||||
""" | |||||
def hash_then_or(hash_name): | |||||
# For now, all the decent hashes have 6-char names, so we can get | |||||
# away with hard-coding space literals. | |||||
return chain([hash_name], repeat(' or')) | |||||
lines = [] | |||||
for hash_name, expecteds in iteritems(self.allowed): | |||||
prefix = hash_then_or(hash_name) | |||||
lines.extend((' Expected %s %s' % (next(prefix), e)) | |||||
for e in expecteds) | |||||
lines.append(' Got %s\n' % | |||||
self.gots[hash_name].hexdigest()) | |||||
prefix = ' or' | |||||
return '\n'.join(lines) | |||||
class UnsupportedPythonVersion(InstallationError): | |||||
"""Unsupported python version according to Requires-Python package | |||||
metadata.""" |
"""Locations where we look for configs, install stuff, etc""" | |||||
from __future__ import absolute_import | |||||
import os | |||||
import os.path | |||||
import platform | |||||
import site | |||||
import sys | |||||
import sysconfig | |||||
from distutils import sysconfig as distutils_sysconfig | |||||
from distutils.command.install import SCHEME_KEYS, install # type: ignore | |||||
from pip._internal.compat import WINDOWS, expanduser | |||||
from pip._internal.utils import appdirs | |||||
# Application Directories | |||||
USER_CACHE_DIR = appdirs.user_cache_dir("pip") | |||||
DELETE_MARKER_MESSAGE = '''\ | |||||
This file is placed here by pip to indicate the source was put | |||||
here by pip. | |||||
Once this package is successfully installed this source code will be | |||||
deleted (unless you remove this file). | |||||
''' | |||||
PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' | |||||
def write_delete_marker_file(directory): | |||||
""" | |||||
Write the pip delete marker file into this directory. | |||||
""" | |||||
filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME) | |||||
with open(filepath, 'w') as marker_fp: | |||||
marker_fp.write(DELETE_MARKER_MESSAGE) | |||||
def running_under_virtualenv(): | |||||
""" | |||||
Return True if we're running inside a virtualenv, False otherwise. | |||||
""" | |||||
if hasattr(sys, 'real_prefix'): | |||||
return True | |||||
elif sys.prefix != getattr(sys, "base_prefix", sys.prefix): | |||||
return True | |||||
return False | |||||
def virtualenv_no_global(): | |||||
""" | |||||
Return True if in a venv and no system site packages. | |||||
""" | |||||
# this mirrors the logic in virtualenv.py for locating the | |||||
# no-global-site-packages.txt file | |||||
site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) | |||||
no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') | |||||
if running_under_virtualenv() and os.path.isfile(no_global_file): | |||||
return True | |||||
if running_under_virtualenv(): | |||||
src_prefix = os.path.join(sys.prefix, 'src') | |||||
else: | |||||
# FIXME: keep src in cwd for now (it is not a temporary folder) | |||||
try: | |||||
src_prefix = os.path.join(os.getcwd(), 'src') | |||||
except OSError: | |||||
# In case the current working directory has been renamed or deleted | |||||
sys.exit( | |||||
"The folder you are executing pip from can no longer be found." | |||||
) | |||||
# under macOS + virtualenv sys.prefix is not properly resolved | |||||
# it is something like /path/to/python/bin/.. | |||||
# Note: using realpath due to tmp dirs on OSX being symlinks | |||||
src_prefix = os.path.abspath(src_prefix) | |||||
# FIXME doesn't account for venv linked to global site-packages | |||||
site_packages = sysconfig.get_path("purelib") | |||||
# This is because of a bug in PyPy's sysconfig module, see | |||||
# https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths | |||||
# for more information. | |||||
if platform.python_implementation().lower() == "pypy": | |||||
site_packages = distutils_sysconfig.get_python_lib() | |||||
try: | |||||
# Use getusersitepackages if this is present, as it ensures that the | |||||
# value is initialised properly. | |||||
user_site = site.getusersitepackages() | |||||
except AttributeError: | |||||
user_site = site.USER_SITE | |||||
user_dir = expanduser('~') | |||||
if WINDOWS: | |||||
bin_py = os.path.join(sys.prefix, 'Scripts') | |||||
bin_user = os.path.join(user_site, 'Scripts') | |||||
# buildout uses 'bin' on Windows too? | |||||
if not os.path.exists(bin_py): | |||||
bin_py = os.path.join(sys.prefix, 'bin') | |||||
bin_user = os.path.join(user_site, 'bin') | |||||
config_basename = 'pip.ini' | |||||
legacy_storage_dir = os.path.join(user_dir, 'pip') | |||||
legacy_config_file = os.path.join( | |||||
legacy_storage_dir, | |||||
config_basename, | |||||
) | |||||
else: | |||||
bin_py = os.path.join(sys.prefix, 'bin') | |||||
bin_user = os.path.join(user_site, 'bin') | |||||
config_basename = 'pip.conf' | |||||
legacy_storage_dir = os.path.join(user_dir, '.pip') | |||||
legacy_config_file = os.path.join( | |||||
legacy_storage_dir, | |||||
config_basename, | |||||
) | |||||
# Forcing to use /usr/local/bin for standard macOS framework installs | |||||
# Also log to ~/Library/Logs/ for use with the Console.app log viewer | |||||
if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': | |||||
bin_py = '/usr/local/bin' | |||||
site_config_files = [ | |||||
os.path.join(path, config_basename) | |||||
for path in appdirs.site_config_dirs('pip') | |||||
] | |||||
venv_config_file = os.path.join(sys.prefix, config_basename) | |||||
new_config_file = os.path.join(appdirs.user_config_dir("pip"), config_basename) | |||||
def distutils_scheme(dist_name, user=False, home=None, root=None, | |||||
isolated=False, prefix=None): | |||||
""" | |||||
Return a distutils install scheme | |||||
""" | |||||
from distutils.dist import Distribution | |||||
scheme = {} | |||||
if isolated: | |||||
extra_dist_args = {"script_args": ["--no-user-cfg"]} | |||||
else: | |||||
extra_dist_args = {} | |||||
dist_args = {'name': dist_name} | |||||
dist_args.update(extra_dist_args) | |||||
d = Distribution(dist_args) | |||||
d.parse_config_files() | |||||
i = d.get_command_obj('install', create=True) | |||||
# NOTE: setting user or home has the side-effect of creating the home dir | |||||
# or user base for installations during finalize_options() | |||||
# ideally, we'd prefer a scheme class that has no side-effects. | |||||
assert not (user and prefix), "user={} prefix={}".format(user, prefix) | |||||
i.user = user or i.user | |||||
if user: | |||||
i.prefix = "" | |||||
i.prefix = prefix or i.prefix | |||||
i.home = home or i.home | |||||
i.root = root or i.root | |||||
i.finalize_options() | |||||
for key in SCHEME_KEYS: | |||||
scheme[key] = getattr(i, 'install_' + key) | |||||
# install_lib specified in setup.cfg should install *everything* | |||||
# into there (i.e. it takes precedence over both purelib and | |||||
# platlib). Note, i.install_lib is *always* set after | |||||
# finalize_options(); we only want to override here if the user | |||||
# has explicitly requested it hence going back to the config | |||||
if 'install_lib' in d.get_option_dict('install'): | |||||
scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) | |||||
if running_under_virtualenv(): | |||||
scheme['headers'] = os.path.join( | |||||
sys.prefix, | |||||
'include', | |||||
'site', | |||||
'python' + sys.version[:3], | |||||
dist_name, | |||||
) | |||||
if root is not None: | |||||
path_no_drive = os.path.splitdrive( | |||||
os.path.abspath(scheme["headers"]))[1] | |||||
scheme["headers"] = os.path.join( | |||||
root, | |||||
path_no_drive[1:], | |||||
) | |||||
return scheme |
from pip._internal.models.index import Index, PyPI | |||||
__all__ = ["Index", "PyPI"] |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
class Index(object): | |||||
def __init__(self, url): | |||||
self.url = url | |||||
self.netloc = urllib_parse.urlsplit(url).netloc | |||||
self.simple_url = self.url_to_path('simple') | |||||
self.pypi_url = self.url_to_path('pypi') | |||||
def url_to_path(self, path): | |||||
return urllib_parse.urljoin(self.url, path) | |||||
PyPI = Index('https://pypi.org/') |
"""Validation of dependencies of packages | |||||
""" | |||||
from collections import namedtuple | |||||
from pip._vendor.packaging.utils import canonicalize_name | |||||
from pip._internal.operations.prepare import make_abstract_dist | |||||
from pip._internal.utils.misc import get_installed_distributions | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from pip._internal.req.req_install import InstallRequirement | |||||
from typing import Any, Dict, Iterator, Set, Tuple, List | |||||
# Shorthands | |||||
PackageSet = Dict[str, 'PackageDetails'] | |||||
Missing = Tuple[str, Any] | |||||
Conflicting = Tuple[str, str, Any] | |||||
MissingDict = Dict[str, List[Missing]] | |||||
ConflictingDict = Dict[str, List[Conflicting]] | |||||
CheckResult = Tuple[MissingDict, ConflictingDict] | |||||
PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) | |||||
def create_package_set_from_installed(**kwargs): | |||||
# type: (**Any) -> PackageSet | |||||
"""Converts a list of distributions into a PackageSet. | |||||
""" | |||||
# Default to using all packages installed on the system | |||||
if kwargs == {}: | |||||
kwargs = {"local_only": False, "skip": ()} | |||||
retval = {} | |||||
for dist in get_installed_distributions(**kwargs): | |||||
name = canonicalize_name(dist.project_name) | |||||
retval[name] = PackageDetails(dist.version, dist.requires()) | |||||
return retval | |||||
def check_package_set(package_set): | |||||
# type: (PackageSet) -> CheckResult | |||||
"""Check if a package set is consistent | |||||
""" | |||||
missing = dict() | |||||
conflicting = dict() | |||||
for package_name in package_set: | |||||
# Info about dependencies of package_name | |||||
missing_deps = set() # type: Set[Missing] | |||||
conflicting_deps = set() # type: Set[Conflicting] | |||||
for req in package_set[package_name].requires: | |||||
name = canonicalize_name(req.project_name) # type: str | |||||
# Check if it's missing | |||||
if name not in package_set: | |||||
missed = True | |||||
if req.marker is not None: | |||||
missed = req.marker.evaluate() | |||||
if missed: | |||||
missing_deps.add((name, req)) | |||||
continue | |||||
# Check if there's a conflict | |||||
version = package_set[name].version # type: str | |||||
if not req.specifier.contains(version, prereleases=True): | |||||
conflicting_deps.add((name, version, req)) | |||||
def str_key(x): | |||||
return str(x) | |||||
if missing_deps: | |||||
missing[package_name] = sorted(missing_deps, key=str_key) | |||||
if conflicting_deps: | |||||
conflicting[package_name] = sorted(conflicting_deps, key=str_key) | |||||
return missing, conflicting | |||||
def check_install_conflicts(to_install): | |||||
# type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult] | |||||
"""For checking if the dependency graph would be consistent after \ | |||||
installing given requirements | |||||
""" | |||||
# Start from the current state | |||||
state = create_package_set_from_installed() | |||||
_simulate_installation_of(to_install, state) | |||||
return state, check_package_set(state) | |||||
# NOTE from @pradyunsg | |||||
# This required a minor update in dependency link handling logic over at | |||||
# operations.prepare.IsSDist.dist() to get it working | |||||
def _simulate_installation_of(to_install, state): | |||||
# type: (List[InstallRequirement], PackageSet) -> None | |||||
"""Computes the version of packages after installing to_install. | |||||
""" | |||||
# Modify it as installing requirement_set would (assuming no errors) | |||||
for inst_req in to_install: | |||||
dist = make_abstract_dist(inst_req).dist(finder=None) | |||||
name = canonicalize_name(dist.key) | |||||
state[name] = PackageDetails(dist.version, dist.requires()) |
from __future__ import absolute_import | |||||
import collections | |||||
import logging | |||||
import os | |||||
import re | |||||
import warnings | |||||
from pip._vendor import pkg_resources, six | |||||
from pip._vendor.packaging.utils import canonicalize_name | |||||
from pip._vendor.pkg_resources import RequirementParseError | |||||
from pip._internal.exceptions import InstallationError | |||||
from pip._internal.req import InstallRequirement | |||||
from pip._internal.req.req_file import COMMENT_RE | |||||
from pip._internal.utils.deprecation import RemovedInPip11Warning | |||||
from pip._internal.utils.misc import ( | |||||
dist_is_editable, get_installed_distributions, | |||||
) | |||||
logger = logging.getLogger(__name__) | |||||
def freeze( | |||||
requirement=None, | |||||
find_links=None, local_only=None, user_only=None, skip_regex=None, | |||||
isolated=False, | |||||
wheel_cache=None, | |||||
exclude_editable=False, | |||||
skip=()): | |||||
find_links = find_links or [] | |||||
skip_match = None | |||||
if skip_regex: | |||||
skip_match = re.compile(skip_regex).search | |||||
dependency_links = [] | |||||
for dist in pkg_resources.working_set: | |||||
if dist.has_metadata('dependency_links.txt'): | |||||
dependency_links.extend( | |||||
dist.get_metadata_lines('dependency_links.txt') | |||||
) | |||||
for link in find_links: | |||||
if '#egg=' in link: | |||||
dependency_links.append(link) | |||||
for link in find_links: | |||||
yield '-f %s' % link | |||||
installations = {} | |||||
for dist in get_installed_distributions(local_only=local_only, | |||||
skip=(), | |||||
user_only=user_only): | |||||
try: | |||||
req = FrozenRequirement.from_dist( | |||||
dist, | |||||
dependency_links | |||||
) | |||||
except RequirementParseError: | |||||
logger.warning( | |||||
"Could not parse requirement: %s", | |||||
dist.project_name | |||||
) | |||||
continue | |||||
if exclude_editable and req.editable: | |||||
continue | |||||
installations[req.name] = req | |||||
if requirement: | |||||
# the options that don't get turned into an InstallRequirement | |||||
# should only be emitted once, even if the same option is in multiple | |||||
# requirements files, so we need to keep track of what has been emitted | |||||
# so that we don't emit it again if it's seen again | |||||
emitted_options = set() | |||||
# keep track of which files a requirement is in so that we can | |||||
# give an accurate warning if a requirement appears multiple times. | |||||
req_files = collections.defaultdict(list) | |||||
for req_file_path in requirement: | |||||
with open(req_file_path) as req_file: | |||||
for line in req_file: | |||||
if (not line.strip() or | |||||
line.strip().startswith('#') or | |||||
(skip_match and skip_match(line)) or | |||||
line.startswith(( | |||||
'-r', '--requirement', | |||||
'-Z', '--always-unzip', | |||||
'-f', '--find-links', | |||||
'-i', '--index-url', | |||||
'--pre', | |||||
'--trusted-host', | |||||
'--process-dependency-links', | |||||
'--extra-index-url'))): | |||||
line = line.rstrip() | |||||
if line not in emitted_options: | |||||
emitted_options.add(line) | |||||
yield line | |||||
continue | |||||
if line.startswith('-e') or line.startswith('--editable'): | |||||
if line.startswith('-e'): | |||||
line = line[2:].strip() | |||||
else: | |||||
line = line[len('--editable'):].strip().lstrip('=') | |||||
line_req = InstallRequirement.from_editable( | |||||
line, | |||||
isolated=isolated, | |||||
wheel_cache=wheel_cache, | |||||
) | |||||
else: | |||||
line_req = InstallRequirement.from_line( | |||||
COMMENT_RE.sub('', line).strip(), | |||||
isolated=isolated, | |||||
wheel_cache=wheel_cache, | |||||
) | |||||
if not line_req.name: | |||||
logger.info( | |||||
"Skipping line in requirement file [%s] because " | |||||
"it's not clear what it would install: %s", | |||||
req_file_path, line.strip(), | |||||
) | |||||
logger.info( | |||||
" (add #egg=PackageName to the URL to avoid" | |||||
" this warning)" | |||||
) | |||||
elif line_req.name not in installations: | |||||
# either it's not installed, or it is installed | |||||
# but has been processed already | |||||
if not req_files[line_req.name]: | |||||
logger.warning( | |||||
"Requirement file [%s] contains %s, but that " | |||||
"package is not installed", | |||||
req_file_path, | |||||
COMMENT_RE.sub('', line).strip(), | |||||
) | |||||
else: | |||||
req_files[line_req.name].append(req_file_path) | |||||
else: | |||||
yield str(installations[line_req.name]).rstrip() | |||||
del installations[line_req.name] | |||||
req_files[line_req.name].append(req_file_path) | |||||
# Warn about requirements that were included multiple times (in a | |||||
# single requirements file or in different requirements files). | |||||
for name, files in six.iteritems(req_files): | |||||
if len(files) > 1: | |||||
logger.warning("Requirement %s included multiple times [%s]", | |||||
name, ', '.join(sorted(set(files)))) | |||||
yield( | |||||
'## The following requirements were added by ' | |||||
'pip freeze:' | |||||
) | |||||
for installation in sorted( | |||||
installations.values(), key=lambda x: x.name.lower()): | |||||
if canonicalize_name(installation.name) not in skip: | |||||
yield str(installation).rstrip() | |||||
class FrozenRequirement(object): | |||||
def __init__(self, name, req, editable, comments=()): | |||||
self.name = name | |||||
self.req = req | |||||
self.editable = editable | |||||
self.comments = comments | |||||
_rev_re = re.compile(r'-r(\d+)$') | |||||
_date_re = re.compile(r'-(20\d\d\d\d\d\d)$') | |||||
@classmethod | |||||
def from_dist(cls, dist, dependency_links): | |||||
location = os.path.normcase(os.path.abspath(dist.location)) | |||||
comments = [] | |||||
from pip._internal.vcs import vcs, get_src_requirement | |||||
if dist_is_editable(dist) and vcs.get_backend_name(location): | |||||
editable = True | |||||
try: | |||||
req = get_src_requirement(dist, location) | |||||
except InstallationError as exc: | |||||
logger.warning( | |||||
"Error when trying to get requirement for VCS system %s, " | |||||
"falling back to uneditable format", exc | |||||
) | |||||
req = None | |||||
if req is None: | |||||
logger.warning( | |||||
'Could not determine repository location of %s', location | |||||
) | |||||
comments.append( | |||||
'## !! Could not determine repository location' | |||||
) | |||||
req = dist.as_requirement() | |||||
editable = False | |||||
else: | |||||
editable = False | |||||
req = dist.as_requirement() | |||||
specs = req.specs | |||||
assert len(specs) == 1 and specs[0][0] in ["==", "==="], \ | |||||
'Expected 1 spec with == or ===; specs = %r; dist = %r' % \ | |||||
(specs, dist) | |||||
version = specs[0][1] | |||||
ver_match = cls._rev_re.search(version) | |||||
date_match = cls._date_re.search(version) | |||||
if ver_match or date_match: | |||||
svn_backend = vcs.get_backend('svn') | |||||
if svn_backend: | |||||
svn_location = svn_backend().get_location( | |||||
dist, | |||||
dependency_links, | |||||
) | |||||
if not svn_location: | |||||
logger.warning( | |||||
'Warning: cannot find svn location for %s', req, | |||||
) | |||||
comments.append( | |||||
'## FIXME: could not find svn URL in dependency_links ' | |||||
'for this package:' | |||||
) | |||||
else: | |||||
warnings.warn( | |||||
"SVN editable detection based on dependency links " | |||||
"will be dropped in the future.", | |||||
RemovedInPip11Warning, | |||||
) | |||||
comments.append( | |||||
'# Installing as editable to satisfy requirement %s:' % | |||||
req | |||||
) | |||||
if ver_match: | |||||
rev = ver_match.group(1) | |||||
else: | |||||
rev = '{%s}' % date_match.group(1) | |||||
editable = True | |||||
req = '%s@%s#egg=%s' % ( | |||||
svn_location, | |||||
rev, | |||||
cls.egg_name(dist) | |||||
) | |||||
return cls(dist.project_name, req, editable, comments) | |||||
@staticmethod | |||||
def egg_name(dist): | |||||
name = dist.egg_name() | |||||
match = re.search(r'-py\d\.\d$', name) | |||||
if match: | |||||
name = name[:match.start()] | |||||
return name | |||||
def __str__(self): | |||||
req = self.req | |||||
if self.editable: | |||||
req = '-e %s' % req | |||||
return '\n'.join(list(self.comments) + [str(req)]) + '\n' |
"""Prepares a distribution for installation | |||||
""" | |||||
import itertools | |||||
import logging | |||||
import os | |||||
import sys | |||||
from copy import copy | |||||
from pip._vendor import pkg_resources, requests | |||||
from pip._internal.build_env import NoOpBuildEnvironment | |||||
from pip._internal.compat import expanduser | |||||
from pip._internal.download import ( | |||||
is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, | |||||
) | |||||
from pip._internal.exceptions import ( | |||||
DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, | |||||
PreviousBuildDirError, VcsHashUnsupported, | |||||
) | |||||
from pip._internal.index import FormatControl | |||||
from pip._internal.req.req_install import InstallRequirement | |||||
from pip._internal.utils.hashes import MissingHashes | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.utils.misc import ( | |||||
call_subprocess, display_path, normalize_path, | |||||
) | |||||
from pip._internal.utils.ui import open_spinner | |||||
from pip._internal.vcs import vcs | |||||
logger = logging.getLogger(__name__) | |||||
def make_abstract_dist(req): | |||||
"""Factory to make an abstract dist object. | |||||
Preconditions: Either an editable req with a source_dir, or satisfied_by or | |||||
a wheel link, or a non-editable req with a source_dir. | |||||
:return: A concrete DistAbstraction. | |||||
""" | |||||
if req.editable: | |||||
return IsSDist(req) | |||||
elif req.link and req.link.is_wheel: | |||||
return IsWheel(req) | |||||
else: | |||||
return IsSDist(req) | |||||
def _install_build_reqs(finder, prefix, build_requirements): | |||||
# NOTE: What follows is not a very good thing. | |||||
# Eventually, this should move into the BuildEnvironment class and | |||||
# that should handle all the isolation and sub-process invocation. | |||||
finder = copy(finder) | |||||
finder.format_control = FormatControl(set(), set([":all:"])) | |||||
urls = [ | |||||
finder.find_requirement( | |||||
InstallRequirement.from_line(r), upgrade=False).url | |||||
for r in build_requirements | |||||
] | |||||
args = [ | |||||
sys.executable, '-m', 'pip', 'install', '--ignore-installed', | |||||
'--no-user', '--prefix', prefix, | |||||
] + list(urls) | |||||
with open_spinner("Installing build dependencies") as spinner: | |||||
call_subprocess(args, show_stdout=False, spinner=spinner) | |||||
class DistAbstraction(object): | |||||
"""Abstracts out the wheel vs non-wheel Resolver.resolve() logic. | |||||
The requirements for anything installable are as follows: | |||||
- we must be able to determine the requirement name | |||||
(or we can't correctly handle the non-upgrade case). | |||||
- we must be able to generate a list of run-time dependencies | |||||
without installing any additional packages (or we would | |||||
have to either burn time by doing temporary isolated installs | |||||
or alternatively violate pips 'don't start installing unless | |||||
all requirements are available' rule - neither of which are | |||||
desirable). | |||||
- for packages with setup requirements, we must also be able | |||||
to determine their requirements without installing additional | |||||
packages (for the same reason as run-time dependencies) | |||||
- we must be able to create a Distribution object exposing the | |||||
above metadata. | |||||
""" | |||||
def __init__(self, req): | |||||
self.req = req | |||||
def dist(self, finder): | |||||
"""Return a setuptools Dist object.""" | |||||
raise NotImplementedError(self.dist) | |||||
def prep_for_dist(self, finder): | |||||
"""Ensure that we can get a Dist for this requirement.""" | |||||
raise NotImplementedError(self.dist) | |||||
class IsWheel(DistAbstraction): | |||||
def dist(self, finder): | |||||
return list(pkg_resources.find_distributions( | |||||
self.req.source_dir))[0] | |||||
def prep_for_dist(self, finder, build_isolation): | |||||
# FIXME:https://github.com/pypa/pip/issues/1112 | |||||
pass | |||||
class IsSDist(DistAbstraction): | |||||
def dist(self, finder): | |||||
dist = self.req.get_dist() | |||||
# FIXME: shouldn't be globally added. | |||||
if finder and dist.has_metadata('dependency_links.txt'): | |||||
finder.add_dependency_links( | |||||
dist.get_metadata_lines('dependency_links.txt') | |||||
) | |||||
return dist | |||||
def prep_for_dist(self, finder, build_isolation): | |||||
# Before calling "setup.py egg_info", we need to set-up the build | |||||
# environment. | |||||
build_requirements, isolate = self.req.get_pep_518_info() | |||||
should_isolate = build_isolation and isolate | |||||
minimum_requirements = ('setuptools', 'wheel') | |||||
missing_requirements = set(minimum_requirements) - set( | |||||
pkg_resources.Requirement(r).key | |||||
for r in build_requirements | |||||
) | |||||
if missing_requirements: | |||||
def format_reqs(rs): | |||||
return ' and '.join(map(repr, sorted(rs))) | |||||
logger.warning( | |||||
"Missing build time requirements in pyproject.toml for %s: " | |||||
"%s.", self.req, format_reqs(missing_requirements) | |||||
) | |||||
logger.warning( | |||||
"This version of pip does not implement PEP 517 so it cannot " | |||||
"build a wheel without %s.", format_reqs(minimum_requirements) | |||||
) | |||||
if should_isolate: | |||||
with self.req.build_env: | |||||
pass | |||||
_install_build_reqs(finder, self.req.build_env.path, | |||||
build_requirements) | |||||
else: | |||||
self.req.build_env = NoOpBuildEnvironment(no_clean=False) | |||||
self.req.run_egg_info() | |||||
self.req.assert_source_matches_version() | |||||
class Installed(DistAbstraction): | |||||
def dist(self, finder): | |||||
return self.req.satisfied_by | |||||
def prep_for_dist(self, finder): | |||||
pass | |||||
class RequirementPreparer(object): | |||||
"""Prepares a Requirement | |||||
""" | |||||
def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, | |||||
progress_bar, build_isolation): | |||||
super(RequirementPreparer, self).__init__() | |||||
self.src_dir = src_dir | |||||
self.build_dir = build_dir | |||||
# Where still packed archives should be written to. If None, they are | |||||
# not saved, and are deleted immediately after unpacking. | |||||
self.download_dir = download_dir | |||||
# Where still-packed .whl files should be written to. If None, they are | |||||
# written to the download_dir parameter. Separate to download_dir to | |||||
# permit only keeping wheel archives for pip wheel. | |||||
if wheel_download_dir: | |||||
wheel_download_dir = normalize_path(wheel_download_dir) | |||||
self.wheel_download_dir = wheel_download_dir | |||||
# NOTE | |||||
# download_dir and wheel_download_dir overlap semantically and may | |||||
# be combined if we're willing to have non-wheel archives present in | |||||
# the wheelhouse output by 'pip wheel'. | |||||
self.progress_bar = progress_bar | |||||
# Is build isolation allowed? | |||||
self.build_isolation = build_isolation | |||||
@property | |||||
def _download_should_save(self): | |||||
# TODO: Modify to reduce indentation needed | |||||
if self.download_dir: | |||||
self.download_dir = expanduser(self.download_dir) | |||||
if os.path.exists(self.download_dir): | |||||
return True | |||||
else: | |||||
logger.critical('Could not find download directory') | |||||
raise InstallationError( | |||||
"Could not find or access download directory '%s'" | |||||
% display_path(self.download_dir)) | |||||
return False | |||||
def prepare_linked_requirement(self, req, session, finder, | |||||
upgrade_allowed, require_hashes): | |||||
"""Prepare a requirement that would be obtained from req.link | |||||
""" | |||||
# TODO: Breakup into smaller functions | |||||
if req.link and req.link.scheme == 'file': | |||||
path = url_to_path(req.link.url) | |||||
logger.info('Processing %s', display_path(path)) | |||||
else: | |||||
logger.info('Collecting %s', req) | |||||
with indent_log(): | |||||
# @@ if filesystem packages are not marked | |||||
# editable in a req, a non deterministic error | |||||
# occurs when the script attempts to unpack the | |||||
# build directory | |||||
req.ensure_has_source_dir(self.build_dir) | |||||
# If a checkout exists, it's unwise to keep going. version | |||||
# inconsistencies are logged later, but do not fail the | |||||
# installation. | |||||
# FIXME: this won't upgrade when there's an existing | |||||
# package unpacked in `req.source_dir` | |||||
# package unpacked in `req.source_dir` | |||||
if os.path.exists(os.path.join(req.source_dir, 'setup.py')): | |||||
raise PreviousBuildDirError( | |||||
"pip can't proceed with requirements '%s' due to a" | |||||
" pre-existing build directory (%s). This is " | |||||
"likely due to a previous installation that failed" | |||||
". pip is being responsible and not assuming it " | |||||
"can delete this. Please delete it and try again." | |||||
% (req, req.source_dir) | |||||
) | |||||
req.populate_link(finder, upgrade_allowed, require_hashes) | |||||
# We can't hit this spot and have populate_link return None. | |||||
# req.satisfied_by is None here (because we're | |||||
# guarded) and upgrade has no impact except when satisfied_by | |||||
# is not None. | |||||
# Then inside find_requirement existing_applicable -> False | |||||
# If no new versions are found, DistributionNotFound is raised, | |||||
# otherwise a result is guaranteed. | |||||
assert req.link | |||||
link = req.link | |||||
# Now that we have the real link, we can tell what kind of | |||||
# requirements we have and raise some more informative errors | |||||
# than otherwise. (For example, we can raise VcsHashUnsupported | |||||
# for a VCS URL rather than HashMissing.) | |||||
if require_hashes: | |||||
# We could check these first 2 conditions inside | |||||
# unpack_url and save repetition of conditions, but then | |||||
# we would report less-useful error messages for | |||||
# unhashable requirements, complaining that there's no | |||||
# hash provided. | |||||
if is_vcs_url(link): | |||||
raise VcsHashUnsupported() | |||||
elif is_file_url(link) and is_dir_url(link): | |||||
raise DirectoryUrlHashUnsupported() | |||||
if not req.original_link and not req.is_pinned: | |||||
# Unpinned packages are asking for trouble when a new | |||||
# version is uploaded. This isn't a security check, but | |||||
# it saves users a surprising hash mismatch in the | |||||
# future. | |||||
# | |||||
# file:/// URLs aren't pinnable, so don't complain | |||||
# about them not being pinned. | |||||
raise HashUnpinned() | |||||
hashes = req.hashes(trust_internet=not require_hashes) | |||||
if require_hashes and not hashes: | |||||
# Known-good hashes are missing for this requirement, so | |||||
# shim it with a facade object that will provoke hash | |||||
# computation and then raise a HashMissing exception | |||||
# showing the user what the hash should be. | |||||
hashes = MissingHashes() | |||||
try: | |||||
download_dir = self.download_dir | |||||
# We always delete unpacked sdists after pip ran. | |||||
autodelete_unpacked = True | |||||
if req.link.is_wheel and self.wheel_download_dir: | |||||
# when doing 'pip wheel` we download wheels to a | |||||
# dedicated dir. | |||||
download_dir = self.wheel_download_dir | |||||
if req.link.is_wheel: | |||||
if download_dir: | |||||
# When downloading, we only unpack wheels to get | |||||
# metadata. | |||||
autodelete_unpacked = True | |||||
else: | |||||
# When installing a wheel, we use the unpacked | |||||
# wheel. | |||||
autodelete_unpacked = False | |||||
unpack_url( | |||||
req.link, req.source_dir, | |||||
download_dir, autodelete_unpacked, | |||||
session=session, hashes=hashes, | |||||
progress_bar=self.progress_bar | |||||
) | |||||
except requests.HTTPError as exc: | |||||
logger.critical( | |||||
'Could not install requirement %s because of error %s', | |||||
req, | |||||
exc, | |||||
) | |||||
raise InstallationError( | |||||
'Could not install requirement %s because of HTTP ' | |||||
'error %s for URL %s' % | |||||
(req, exc, req.link) | |||||
) | |||||
abstract_dist = make_abstract_dist(req) | |||||
abstract_dist.prep_for_dist(finder, self.build_isolation) | |||||
if self._download_should_save: | |||||
# Make a .zip of the source_dir we already created. | |||||
if req.link.scheme in vcs.all_schemes: | |||||
req.archive(self.download_dir) | |||||
return abstract_dist | |||||
def prepare_editable_requirement(self, req, require_hashes, use_user_site, | |||||
finder): | |||||
"""Prepare an editable requirement | |||||
""" | |||||
assert req.editable, "cannot prepare a non-editable req as editable" | |||||
logger.info('Obtaining %s', req) | |||||
with indent_log(): | |||||
if require_hashes: | |||||
raise InstallationError( | |||||
'The editable requirement %s cannot be installed when ' | |||||
'requiring hashes, because there is no single file to ' | |||||
'hash.' % req | |||||
) | |||||
req.ensure_has_source_dir(self.src_dir) | |||||
req.update_editable(not self._download_should_save) | |||||
abstract_dist = make_abstract_dist(req) | |||||
abstract_dist.prep_for_dist(finder, self.build_isolation) | |||||
if self._download_should_save: | |||||
req.archive(self.download_dir) | |||||
req.check_if_exists(use_user_site) | |||||
return abstract_dist | |||||
def prepare_installed_requirement(self, req, require_hashes, skip_reason): | |||||
"""Prepare an already-installed requirement | |||||
""" | |||||
assert req.satisfied_by, "req should have been satisfied but isn't" | |||||
assert skip_reason is not None, ( | |||||
"did not get skip reason skipped but req.satisfied_by " | |||||
"is set to %r" % (req.satisfied_by,) | |||||
) | |||||
logger.info( | |||||
'Requirement %s: %s (%s)', | |||||
skip_reason, req, req.satisfied_by.version | |||||
) | |||||
with indent_log(): | |||||
if require_hashes: | |||||
logger.debug( | |||||
'Since it is already installed, we are trusting this ' | |||||
'package without checking its hash. To ensure a ' | |||||
'completely repeatable environment, install into an ' | |||||
'empty virtualenv.' | |||||
) | |||||
abstract_dist = Installed(req) | |||||
return abstract_dist |
"""Generate and work with PEP 425 Compatibility Tags.""" | |||||
from __future__ import absolute_import | |||||
import distutils.util | |||||
import logging | |||||
import platform | |||||
import re | |||||
import sys | |||||
import sysconfig | |||||
import warnings | |||||
from collections import OrderedDict | |||||
import pip._internal.utils.glibc | |||||
logger = logging.getLogger(__name__) | |||||
_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') | |||||
def get_config_var(var): | |||||
try: | |||||
return sysconfig.get_config_var(var) | |||||
except IOError as e: # Issue #1074 | |||||
warnings.warn("{}".format(e), RuntimeWarning) | |||||
return None | |||||
def get_abbr_impl(): | |||||
"""Return abbreviated implementation name.""" | |||||
if hasattr(sys, 'pypy_version_info'): | |||||
pyimpl = 'pp' | |||||
elif sys.platform.startswith('java'): | |||||
pyimpl = 'jy' | |||||
elif sys.platform == 'cli': | |||||
pyimpl = 'ip' | |||||
else: | |||||
pyimpl = 'cp' | |||||
return pyimpl | |||||
def get_impl_ver(): | |||||
"""Return implementation version.""" | |||||
impl_ver = get_config_var("py_version_nodot") | |||||
if not impl_ver or get_abbr_impl() == 'pp': | |||||
impl_ver = ''.join(map(str, get_impl_version_info())) | |||||
return impl_ver | |||||
def get_impl_version_info(): | |||||
"""Return sys.version_info-like tuple for use in decrementing the minor | |||||
version.""" | |||||
if get_abbr_impl() == 'pp': | |||||
# as per https://github.com/pypa/pip/issues/2882 | |||||
return (sys.version_info[0], sys.pypy_version_info.major, | |||||
sys.pypy_version_info.minor) | |||||
else: | |||||
return sys.version_info[0], sys.version_info[1] | |||||
def get_impl_tag(): | |||||
""" | |||||
Returns the Tag for this specific implementation. | |||||
""" | |||||
return "{}{}".format(get_abbr_impl(), get_impl_ver()) | |||||
def get_flag(var, fallback, expected=True, warn=True): | |||||
"""Use a fallback method for determining SOABI flags if the needed config | |||||
var is unset or unavailable.""" | |||||
val = get_config_var(var) | |||||
if val is None: | |||||
if warn: | |||||
logger.debug("Config variable '%s' is unset, Python ABI tag may " | |||||
"be incorrect", var) | |||||
return fallback() | |||||
return val == expected | |||||
def get_abi_tag(): | |||||
"""Return the ABI tag based on SOABI (if available) or emulate SOABI | |||||
(CPython 2, PyPy).""" | |||||
soabi = get_config_var('SOABI') | |||||
impl = get_abbr_impl() | |||||
if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'): | |||||
d = '' | |||||
m = '' | |||||
u = '' | |||||
if get_flag('Py_DEBUG', | |||||
lambda: hasattr(sys, 'gettotalrefcount'), | |||||
warn=(impl == 'cp')): | |||||
d = 'd' | |||||
if get_flag('WITH_PYMALLOC', | |||||
lambda: impl == 'cp', | |||||
warn=(impl == 'cp')): | |||||
m = 'm' | |||||
if get_flag('Py_UNICODE_SIZE', | |||||
lambda: sys.maxunicode == 0x10ffff, | |||||
expected=4, | |||||
warn=(impl == 'cp' and | |||||
sys.version_info < (3, 3))) \ | |||||
and sys.version_info < (3, 3): | |||||
u = 'u' | |||||
abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u) | |||||
elif soabi and soabi.startswith('cpython-'): | |||||
abi = 'cp' + soabi.split('-')[1] | |||||
elif soabi: | |||||
abi = soabi.replace('.', '_').replace('-', '_') | |||||
else: | |||||
abi = None | |||||
return abi | |||||
def _is_running_32bit(): | |||||
return sys.maxsize == 2147483647 | |||||
def get_platform(): | |||||
"""Return our platform name 'win32', 'linux_x86_64'""" | |||||
if sys.platform == 'darwin': | |||||
# distutils.util.get_platform() returns the release based on the value | |||||
# of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may | |||||
# be significantly older than the user's current machine. | |||||
release, _, machine = platform.mac_ver() | |||||
split_ver = release.split('.') | |||||
if machine == "x86_64" and _is_running_32bit(): | |||||
machine = "i386" | |||||
elif machine == "ppc64" and _is_running_32bit(): | |||||
machine = "ppc" | |||||
return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine) | |||||
# XXX remove distutils dependency | |||||
result = distutils.util.get_platform().replace('.', '_').replace('-', '_') | |||||
if result == "linux_x86_64" and _is_running_32bit(): | |||||
# 32 bit Python program (running on a 64 bit Linux): pip should only | |||||
# install and run 32 bit compiled extensions in that case. | |||||
result = "linux_i686" | |||||
return result | |||||
def is_manylinux1_compatible(): | |||||
# Only Linux, and only x86-64 / i686 | |||||
if get_platform() not in {"linux_x86_64", "linux_i686"}: | |||||
return False | |||||
# Check for presence of _manylinux module | |||||
try: | |||||
import _manylinux | |||||
return bool(_manylinux.manylinux1_compatible) | |||||
except (ImportError, AttributeError): | |||||
# Fall through to heuristic check below | |||||
pass | |||||
# Check glibc version. CentOS 5 uses glibc 2.5. | |||||
return pip._internal.utils.glibc.have_compatible_glibc(2, 5) | |||||
def get_darwin_arches(major, minor, machine): | |||||
"""Return a list of supported arches (including group arches) for | |||||
the given major, minor and machine architecture of an macOS machine. | |||||
""" | |||||
arches = [] | |||||
def _supports_arch(major, minor, arch): | |||||
# Looking at the application support for macOS versions in the chart | |||||
# provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears | |||||
# our timeline looks roughly like: | |||||
# | |||||
# 10.0 - Introduces ppc support. | |||||
# 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64 | |||||
# and x86_64 support is CLI only, and cannot be used for GUI | |||||
# applications. | |||||
# 10.5 - Extends ppc64 and x86_64 support to cover GUI applications. | |||||
# 10.6 - Drops support for ppc64 | |||||
# 10.7 - Drops support for ppc | |||||
# | |||||
# Given that we do not know if we're installing a CLI or a GUI | |||||
# application, we must be conservative and assume it might be a GUI | |||||
# application and behave as if ppc64 and x86_64 support did not occur | |||||
# until 10.5. | |||||
# | |||||
# Note: The above information is taken from the "Application support" | |||||
# column in the chart not the "Processor support" since I believe | |||||
# that we care about what instruction sets an application can use | |||||
# not which processors the OS supports. | |||||
if arch == 'ppc': | |||||
return (major, minor) <= (10, 5) | |||||
if arch == 'ppc64': | |||||
return (major, minor) == (10, 5) | |||||
if arch == 'i386': | |||||
return (major, minor) >= (10, 4) | |||||
if arch == 'x86_64': | |||||
return (major, minor) >= (10, 5) | |||||
if arch in groups: | |||||
for garch in groups[arch]: | |||||
if _supports_arch(major, minor, garch): | |||||
return True | |||||
return False | |||||
groups = OrderedDict([ | |||||
("fat", ("i386", "ppc")), | |||||
("intel", ("x86_64", "i386")), | |||||
("fat64", ("x86_64", "ppc64")), | |||||
("fat32", ("x86_64", "i386", "ppc")), | |||||
]) | |||||
if _supports_arch(major, minor, machine): | |||||
arches.append(machine) | |||||
for garch in groups: | |||||
if machine in groups[garch] and _supports_arch(major, minor, garch): | |||||
arches.append(garch) | |||||
arches.append('universal') | |||||
return arches | |||||
def get_supported(versions=None, noarch=False, platform=None, | |||||
impl=None, abi=None): | |||||
"""Return a list of supported tags for each version specified in | |||||
`versions`. | |||||
:param versions: a list of string versions, of the form ["33", "32"], | |||||
or None. The first version will be assumed to support our ABI. | |||||
:param platform: specify the exact platform you want valid | |||||
tags for, or None. If None, use the local system platform. | |||||
:param impl: specify the exact implementation you want valid | |||||
tags for, or None. If None, use the local interpreter impl. | |||||
:param abi: specify the exact abi you want valid | |||||
tags for, or None. If None, use the local interpreter abi. | |||||
""" | |||||
supported = [] | |||||
# Versions must be given with respect to the preference | |||||
if versions is None: | |||||
versions = [] | |||||
version_info = get_impl_version_info() | |||||
major = version_info[:-1] | |||||
# Support all previous minor Python versions. | |||||
for minor in range(version_info[-1], -1, -1): | |||||
versions.append(''.join(map(str, major + (minor,)))) | |||||
impl = impl or get_abbr_impl() | |||||
abis = [] | |||||
abi = abi or get_abi_tag() | |||||
if abi: | |||||
abis[0:0] = [abi] | |||||
abi3s = set() | |||||
import imp | |||||
for suffix in imp.get_suffixes(): | |||||
if suffix[0].startswith('.abi'): | |||||
abi3s.add(suffix[0].split('.', 2)[1]) | |||||
abis.extend(sorted(list(abi3s))) | |||||
abis.append('none') | |||||
if not noarch: | |||||
arch = platform or get_platform() | |||||
if arch.startswith('macosx'): | |||||
# support macosx-10.6-intel on macosx-10.9-x86_64 | |||||
match = _osx_arch_pat.match(arch) | |||||
if match: | |||||
name, major, minor, actual_arch = match.groups() | |||||
tpl = '{}_{}_%i_%s'.format(name, major) | |||||
arches = [] | |||||
for m in reversed(range(int(minor) + 1)): | |||||
for a in get_darwin_arches(int(major), m, actual_arch): | |||||
arches.append(tpl % (m, a)) | |||||
else: | |||||
# arch pattern didn't match (?!) | |||||
arches = [arch] | |||||
elif platform is None and is_manylinux1_compatible(): | |||||
arches = [arch.replace('linux', 'manylinux1'), arch] | |||||
else: | |||||
arches = [arch] | |||||
# Current version, current API (built specifically for our Python): | |||||
for abi in abis: | |||||
for arch in arches: | |||||
supported.append(('%s%s' % (impl, versions[0]), abi, arch)) | |||||
# abi3 modules compatible with older version of Python | |||||
for version in versions[1:]: | |||||
# abi3 was introduced in Python 3.2 | |||||
if version in {'31', '30'}: | |||||
break | |||||
for abi in abi3s: # empty set if not Python 3 | |||||
for arch in arches: | |||||
supported.append(("%s%s" % (impl, version), abi, arch)) | |||||
# Has binaries, does not use the Python API: | |||||
for arch in arches: | |||||
supported.append(('py%s' % (versions[0][0]), 'none', arch)) | |||||
# No abi / arch, but requires our implementation: | |||||
supported.append(('%s%s' % (impl, versions[0]), 'none', 'any')) | |||||
# Tagged specifically as being cross-version compatible | |||||
# (with just the major version specified) | |||||
supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any')) | |||||
# No abi / arch, generic Python | |||||
for i, version in enumerate(versions): | |||||
supported.append(('py%s' % (version,), 'none', 'any')) | |||||
if i == 0: | |||||
supported.append(('py%s' % (version[0]), 'none', 'any')) | |||||
return supported | |||||
implementation_tag = get_impl_tag() |
from __future__ import absolute_import | |||||
import logging | |||||
from .req_install import InstallRequirement | |||||
from .req_set import RequirementSet | |||||
from .req_file import parse_requirements | |||||
from pip._internal.utils.logging import indent_log | |||||
__all__ = [ | |||||
"RequirementSet", "InstallRequirement", | |||||
"parse_requirements", "install_given_reqs", | |||||
] | |||||
logger = logging.getLogger(__name__) | |||||
def install_given_reqs(to_install, install_options, global_options=(), | |||||
*args, **kwargs): | |||||
""" | |||||
Install everything in the given list. | |||||
(to be called after having downloaded and unpacked the packages) | |||||
""" | |||||
if to_install: | |||||
logger.info( | |||||
'Installing collected packages: %s', | |||||
', '.join([req.name for req in to_install]), | |||||
) | |||||
with indent_log(): | |||||
for requirement in to_install: | |||||
if requirement.conflicts_with: | |||||
logger.info( | |||||
'Found existing installation: %s', | |||||
requirement.conflicts_with, | |||||
) | |||||
with indent_log(): | |||||
uninstalled_pathset = requirement.uninstall( | |||||
auto_confirm=True | |||||
) | |||||
try: | |||||
requirement.install( | |||||
install_options, | |||||
global_options, | |||||
*args, | |||||
**kwargs | |||||
) | |||||
except: | |||||
should_rollback = ( | |||||
requirement.conflicts_with and | |||||
not requirement.install_succeeded | |||||
) | |||||
# if install did not succeed, rollback previous uninstall | |||||
if should_rollback: | |||||
uninstalled_pathset.rollback() | |||||
raise | |||||
else: | |||||
should_commit = ( | |||||
requirement.conflicts_with and | |||||
requirement.install_succeeded | |||||
) | |||||
if should_commit: | |||||
uninstalled_pathset.commit() | |||||
requirement.remove_temporary_source() | |||||
return to_install |
""" | |||||
Requirements file parsing | |||||
""" | |||||
from __future__ import absolute_import | |||||
import optparse | |||||
import os | |||||
import re | |||||
import shlex | |||||
import sys | |||||
from pip._vendor.six.moves import filterfalse | |||||
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
from pip._internal import cmdoptions | |||||
from pip._internal.download import get_file_content | |||||
from pip._internal.exceptions import RequirementsFileParseError | |||||
from pip._internal.req.req_install import InstallRequirement | |||||
__all__ = ['parse_requirements'] | |||||
SCHEME_RE = re.compile(r'^(http|https|file):', re.I) | |||||
COMMENT_RE = re.compile(r'(^|\s)+#.*$') | |||||
# Matches environment variable-style values in '${MY_VARIABLE_1}' with the | |||||
# variable name consisting of only uppercase letters, digits or the '_' | |||||
# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1, | |||||
# 2013 Edition. | |||||
ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})') | |||||
SUPPORTED_OPTIONS = [ | |||||
cmdoptions.constraints, | |||||
cmdoptions.editable, | |||||
cmdoptions.requirements, | |||||
cmdoptions.no_index, | |||||
cmdoptions.index_url, | |||||
cmdoptions.find_links, | |||||
cmdoptions.extra_index_url, | |||||
cmdoptions.always_unzip, | |||||
cmdoptions.no_binary, | |||||
cmdoptions.only_binary, | |||||
cmdoptions.pre, | |||||
cmdoptions.process_dependency_links, | |||||
cmdoptions.trusted_host, | |||||
cmdoptions.require_hashes, | |||||
] | |||||
# options to be passed to requirements | |||||
SUPPORTED_OPTIONS_REQ = [ | |||||
cmdoptions.install_options, | |||||
cmdoptions.global_options, | |||||
cmdoptions.hash, | |||||
] | |||||
# the 'dest' string values | |||||
SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] | |||||
def parse_requirements(filename, finder=None, comes_from=None, options=None, | |||||
session=None, constraint=False, wheel_cache=None): | |||||
"""Parse a requirements file and yield InstallRequirement instances. | |||||
:param filename: Path or url of requirements file. | |||||
:param finder: Instance of pip.index.PackageFinder. | |||||
:param comes_from: Origin description of requirements. | |||||
:param options: cli options. | |||||
:param session: Instance of pip.download.PipSession. | |||||
:param constraint: If true, parsing a constraint file rather than | |||||
requirements file. | |||||
:param wheel_cache: Instance of pip.wheel.WheelCache | |||||
""" | |||||
if session is None: | |||||
raise TypeError( | |||||
"parse_requirements() missing 1 required keyword argument: " | |||||
"'session'" | |||||
) | |||||
_, content = get_file_content( | |||||
filename, comes_from=comes_from, session=session | |||||
) | |||||
lines_enum = preprocess(content, options) | |||||
for line_number, line in lines_enum: | |||||
req_iter = process_line(line, filename, line_number, finder, | |||||
comes_from, options, session, wheel_cache, | |||||
constraint=constraint) | |||||
for req in req_iter: | |||||
yield req | |||||
def preprocess(content, options): | |||||
"""Split, filter, and join lines, and return a line iterator | |||||
:param content: the content of the requirements file | |||||
:param options: cli options | |||||
""" | |||||
lines_enum = enumerate(content.splitlines(), start=1) | |||||
lines_enum = join_lines(lines_enum) | |||||
lines_enum = ignore_comments(lines_enum) | |||||
lines_enum = skip_regex(lines_enum, options) | |||||
lines_enum = expand_env_variables(lines_enum) | |||||
return lines_enum | |||||
def process_line(line, filename, line_number, finder=None, comes_from=None, | |||||
options=None, session=None, wheel_cache=None, | |||||
constraint=False): | |||||
"""Process a single requirements line; This can result in creating/yielding | |||||
requirements, or updating the finder. | |||||
For lines that contain requirements, the only options that have an effect | |||||
are from SUPPORTED_OPTIONS_REQ, and they are scoped to the | |||||
requirement. Other options from SUPPORTED_OPTIONS may be present, but are | |||||
ignored. | |||||
For lines that do not contain requirements, the only options that have an | |||||
effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may | |||||
be present, but are ignored. These lines may contain multiple options | |||||
(although our docs imply only one is supported), and all our parsed and | |||||
affect the finder. | |||||
:param constraint: If True, parsing a constraints file. | |||||
:param options: OptionParser options that we may update | |||||
""" | |||||
parser = build_parser(line) | |||||
defaults = parser.get_default_values() | |||||
defaults.index_url = None | |||||
if finder: | |||||
# `finder.format_control` will be updated during parsing | |||||
defaults.format_control = finder.format_control | |||||
args_str, options_str = break_args_options(line) | |||||
if sys.version_info < (2, 7, 3): | |||||
# Prior to 2.7.3, shlex cannot deal with unicode entries | |||||
options_str = options_str.encode('utf8') | |||||
opts, _ = parser.parse_args(shlex.split(options_str), defaults) | |||||
# preserve for the nested code path | |||||
line_comes_from = '%s %s (line %s)' % ( | |||||
'-c' if constraint else '-r', filename, line_number, | |||||
) | |||||
# yield a line requirement | |||||
if args_str: | |||||
isolated = options.isolated_mode if options else False | |||||
if options: | |||||
cmdoptions.check_install_build_global(options, opts) | |||||
# get the options that apply to requirements | |||||
req_options = {} | |||||
for dest in SUPPORTED_OPTIONS_REQ_DEST: | |||||
if dest in opts.__dict__ and opts.__dict__[dest]: | |||||
req_options[dest] = opts.__dict__[dest] | |||||
yield InstallRequirement.from_line( | |||||
args_str, line_comes_from, constraint=constraint, | |||||
isolated=isolated, options=req_options, wheel_cache=wheel_cache | |||||
) | |||||
# yield an editable requirement | |||||
elif opts.editables: | |||||
isolated = options.isolated_mode if options else False | |||||
yield InstallRequirement.from_editable( | |||||
opts.editables[0], comes_from=line_comes_from, | |||||
constraint=constraint, isolated=isolated, wheel_cache=wheel_cache | |||||
) | |||||
# parse a nested requirements file | |||||
elif opts.requirements or opts.constraints: | |||||
if opts.requirements: | |||||
req_path = opts.requirements[0] | |||||
nested_constraint = False | |||||
else: | |||||
req_path = opts.constraints[0] | |||||
nested_constraint = True | |||||
# original file is over http | |||||
if SCHEME_RE.search(filename): | |||||
# do a url join so relative paths work | |||||
req_path = urllib_parse.urljoin(filename, req_path) | |||||
# original file and nested file are paths | |||||
elif not SCHEME_RE.search(req_path): | |||||
# do a join so relative paths work | |||||
req_path = os.path.join(os.path.dirname(filename), req_path) | |||||
# TODO: Why not use `comes_from='-r {} (line {})'` here as well? | |||||
parser = parse_requirements( | |||||
req_path, finder, comes_from, options, session, | |||||
constraint=nested_constraint, wheel_cache=wheel_cache | |||||
) | |||||
for req in parser: | |||||
yield req | |||||
# percolate hash-checking option upward | |||||
elif opts.require_hashes: | |||||
options.require_hashes = opts.require_hashes | |||||
# set finder options | |||||
elif finder: | |||||
if opts.index_url: | |||||
finder.index_urls = [opts.index_url] | |||||
if opts.no_index is True: | |||||
finder.index_urls = [] | |||||
if opts.extra_index_urls: | |||||
finder.index_urls.extend(opts.extra_index_urls) | |||||
if opts.find_links: | |||||
# FIXME: it would be nice to keep track of the source | |||||
# of the find_links: support a find-links local path | |||||
# relative to a requirements file. | |||||
value = opts.find_links[0] | |||||
req_dir = os.path.dirname(os.path.abspath(filename)) | |||||
relative_to_reqs_file = os.path.join(req_dir, value) | |||||
if os.path.exists(relative_to_reqs_file): | |||||
value = relative_to_reqs_file | |||||
finder.find_links.append(value) | |||||
if opts.pre: | |||||
finder.allow_all_prereleases = True | |||||
if opts.process_dependency_links: | |||||
finder.process_dependency_links = True | |||||
if opts.trusted_hosts: | |||||
finder.secure_origins.extend( | |||||
("*", host, "*") for host in opts.trusted_hosts) | |||||
def break_args_options(line): | |||||
"""Break up the line into an args and options string. We only want to shlex | |||||
(and then optparse) the options, not the args. args can contain markers | |||||
which are corrupted by shlex. | |||||
""" | |||||
tokens = line.split(' ') | |||||
args = [] | |||||
options = tokens[:] | |||||
for token in tokens: | |||||
if token.startswith('-') or token.startswith('--'): | |||||
break | |||||
else: | |||||
args.append(token) | |||||
options.pop(0) | |||||
return ' '.join(args), ' '.join(options) | |||||
def build_parser(line): | |||||
""" | |||||
Return a parser for parsing requirement lines | |||||
""" | |||||
parser = optparse.OptionParser(add_help_option=False) | |||||
option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ | |||||
for option_factory in option_factories: | |||||
option = option_factory() | |||||
parser.add_option(option) | |||||
# By default optparse sys.exits on parsing errors. We want to wrap | |||||
# that in our own exception. | |||||
def parser_exit(self, msg): | |||||
# add offending line | |||||
msg = 'Invalid requirement: %s\n%s' % (line, msg) | |||||
raise RequirementsFileParseError(msg) | |||||
parser.exit = parser_exit | |||||
return parser | |||||
def join_lines(lines_enum): | |||||
"""Joins a line ending in '\' with the previous line (except when following | |||||
comments). The joined line takes on the index of the first line. | |||||
""" | |||||
primary_line_number = None | |||||
new_line = [] | |||||
for line_number, line in lines_enum: | |||||
if not line.endswith('\\') or COMMENT_RE.match(line): | |||||
if COMMENT_RE.match(line): | |||||
# this ensures comments are always matched later | |||||
line = ' ' + line | |||||
if new_line: | |||||
new_line.append(line) | |||||
yield primary_line_number, ''.join(new_line) | |||||
new_line = [] | |||||
else: | |||||
yield line_number, line | |||||
else: | |||||
if not new_line: | |||||
primary_line_number = line_number | |||||
new_line.append(line.strip('\\')) | |||||
# last line contains \ | |||||
if new_line: | |||||
yield primary_line_number, ''.join(new_line) | |||||
# TODO: handle space after '\'. | |||||
def ignore_comments(lines_enum): | |||||
""" | |||||
Strips comments and filter empty lines. | |||||
""" | |||||
for line_number, line in lines_enum: | |||||
line = COMMENT_RE.sub('', line) | |||||
line = line.strip() | |||||
if line: | |||||
yield line_number, line | |||||
def skip_regex(lines_enum, options): | |||||
""" | |||||
Skip lines that match '--skip-requirements-regex' pattern | |||||
Note: the regex pattern is only built once | |||||
""" | |||||
skip_regex = options.skip_requirements_regex if options else None | |||||
if skip_regex: | |||||
pattern = re.compile(skip_regex) | |||||
lines_enum = filterfalse(lambda e: pattern.search(e[1]), lines_enum) | |||||
return lines_enum | |||||
def expand_env_variables(lines_enum): | |||||
"""Replace all environment variables that can be retrieved via `os.getenv`. | |||||
The only allowed format for environment variables defined in the | |||||
requirement file is `${MY_VARIABLE_1}` to ensure two things: | |||||
1. Strings that contain a `$` aren't accidentally (partially) expanded. | |||||
2. Ensure consistency across platforms for requirement files. | |||||
These points are the result of a discusssion on the `github pull | |||||
request #3514 <https://github.com/pypa/pip/pull/3514>`_. | |||||
Valid characters in variable names follow the `POSIX standard | |||||
<http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited | |||||
to uppercase letter, digits and the `_` (underscore). | |||||
""" | |||||
for line_number, line in lines_enum: | |||||
for env_var, var_name in ENV_VAR_RE.findall(line): | |||||
value = os.getenv(var_name) | |||||
if not value: | |||||
continue | |||||
line = line.replace(env_var, value) | |||||
yield line_number, line |
from __future__ import absolute_import | |||||
import logging | |||||
from collections import OrderedDict | |||||
from pip._internal.exceptions import InstallationError | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.wheel import Wheel | |||||
logger = logging.getLogger(__name__) | |||||
class RequirementSet(object): | |||||
def __init__(self, require_hashes=False): | |||||
"""Create a RequirementSet. | |||||
:param wheel_cache: The pip wheel cache, for passing to | |||||
InstallRequirement. | |||||
""" | |||||
self.requirements = OrderedDict() | |||||
self.require_hashes = require_hashes | |||||
# Mapping of alias: real_name | |||||
self.requirement_aliases = {} | |||||
self.unnamed_requirements = [] | |||||
self.successfully_downloaded = [] | |||||
self.reqs_to_cleanup = [] | |||||
def __str__(self): | |||||
reqs = [req for req in self.requirements.values() | |||||
if not req.comes_from] | |||||
reqs.sort(key=lambda req: req.name.lower()) | |||||
return ' '.join([str(req.req) for req in reqs]) | |||||
def __repr__(self): | |||||
reqs = [req for req in self.requirements.values()] | |||||
reqs.sort(key=lambda req: req.name.lower()) | |||||
reqs_str = ', '.join([str(req.req) for req in reqs]) | |||||
return ('<%s object; %d requirement(s): %s>' | |||||
% (self.__class__.__name__, len(reqs), reqs_str)) | |||||
def add_requirement(self, install_req, parent_req_name=None, | |||||
extras_requested=None): | |||||
"""Add install_req as a requirement to install. | |||||
:param parent_req_name: The name of the requirement that needed this | |||||
added. The name is used because when multiple unnamed requirements | |||||
resolve to the same name, we could otherwise end up with dependency | |||||
links that point outside the Requirements set. parent_req must | |||||
already be added. Note that None implies that this is a user | |||||
supplied requirement, vs an inferred one. | |||||
:param extras_requested: an iterable of extras used to evaluate the | |||||
environment markers. | |||||
:return: Additional requirements to scan. That is either [] if | |||||
the requirement is not applicable, or [install_req] if the | |||||
requirement is applicable and has just been added. | |||||
""" | |||||
name = install_req.name | |||||
if not install_req.match_markers(extras_requested): | |||||
logger.info("Ignoring %s: markers '%s' don't match your " | |||||
"environment", install_req.name, | |||||
install_req.markers) | |||||
return [], None | |||||
# This check has to come after we filter requirements with the | |||||
# environment markers. | |||||
if install_req.link and install_req.link.is_wheel: | |||||
wheel = Wheel(install_req.link.filename) | |||||
if not wheel.supported(): | |||||
raise InstallationError( | |||||
"%s is not a supported wheel on this platform." % | |||||
wheel.filename | |||||
) | |||||
# This next bit is really a sanity check. | |||||
assert install_req.is_direct == (parent_req_name is None), ( | |||||
"a direct req shouldn't have a parent and also, " | |||||
"a non direct req should have a parent" | |||||
) | |||||
if not name: | |||||
# url or path requirement w/o an egg fragment | |||||
self.unnamed_requirements.append(install_req) | |||||
return [install_req], None | |||||
else: | |||||
try: | |||||
existing_req = self.get_requirement(name) | |||||
except KeyError: | |||||
existing_req = None | |||||
if (parent_req_name is None and existing_req and not | |||||
existing_req.constraint and | |||||
existing_req.extras == install_req.extras and not | |||||
existing_req.req.specifier == install_req.req.specifier): | |||||
raise InstallationError( | |||||
'Double requirement given: %s (already in %s, name=%r)' | |||||
% (install_req, existing_req, name)) | |||||
if not existing_req: | |||||
# Add requirement | |||||
self.requirements[name] = install_req | |||||
# FIXME: what about other normalizations? E.g., _ vs. -? | |||||
if name.lower() != name: | |||||
self.requirement_aliases[name.lower()] = name | |||||
result = [install_req] | |||||
else: | |||||
# Assume there's no need to scan, and that we've already | |||||
# encountered this for scanning. | |||||
result = [] | |||||
if not install_req.constraint and existing_req.constraint: | |||||
if (install_req.link and not (existing_req.link and | |||||
install_req.link.path == existing_req.link.path)): | |||||
self.reqs_to_cleanup.append(install_req) | |||||
raise InstallationError( | |||||
"Could not satisfy constraints for '%s': " | |||||
"installation from path or url cannot be " | |||||
"constrained to a version" % name, | |||||
) | |||||
# If we're now installing a constraint, mark the existing | |||||
# object for real installation. | |||||
existing_req.constraint = False | |||||
existing_req.extras = tuple( | |||||
sorted(set(existing_req.extras).union( | |||||
set(install_req.extras)))) | |||||
logger.debug("Setting %s extras to: %s", | |||||
existing_req, existing_req.extras) | |||||
# And now we need to scan this. | |||||
result = [existing_req] | |||||
# Canonicalise to the already-added object for the backref | |||||
# check below. | |||||
install_req = existing_req | |||||
# We return install_req here to allow for the caller to add it to | |||||
# the dependency information for the parent package. | |||||
return result, install_req | |||||
def has_requirement(self, project_name): | |||||
name = project_name.lower() | |||||
if (name in self.requirements and | |||||
not self.requirements[name].constraint or | |||||
name in self.requirement_aliases and | |||||
not self.requirements[self.requirement_aliases[name]].constraint): | |||||
return True | |||||
return False | |||||
@property | |||||
def has_requirements(self): | |||||
return list(req for req in self.requirements.values() if not | |||||
req.constraint) or self.unnamed_requirements | |||||
def get_requirement(self, project_name): | |||||
for name in project_name, project_name.lower(): | |||||
if name in self.requirements: | |||||
return self.requirements[name] | |||||
if name in self.requirement_aliases: | |||||
return self.requirements[self.requirement_aliases[name]] | |||||
raise KeyError("No project with the name %r" % project_name) | |||||
def cleanup_files(self): | |||||
"""Clean up files, remove builds.""" | |||||
logger.debug('Cleaning up...') | |||||
with indent_log(): | |||||
for req in self.reqs_to_cleanup: | |||||
req.remove_temporary_source() |
from __future__ import absolute_import | |||||
import csv | |||||
import functools | |||||
import logging | |||||
import os | |||||
import sys | |||||
import sysconfig | |||||
from pip._vendor import pkg_resources | |||||
from pip._internal.compat import WINDOWS, cache_from_source, uses_pycache | |||||
from pip._internal.exceptions import UninstallationError | |||||
from pip._internal.locations import bin_py, bin_user | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.utils.misc import ( | |||||
FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, | |||||
normalize_path, renames, | |||||
) | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
logger = logging.getLogger(__name__) | |||||
def _script_names(dist, script_name, is_gui): | |||||
"""Create the fully qualified name of the files created by | |||||
{console,gui}_scripts for the given ``dist``. | |||||
Returns the list of file names | |||||
""" | |||||
if dist_in_usersite(dist): | |||||
bin_dir = bin_user | |||||
else: | |||||
bin_dir = bin_py | |||||
exe_name = os.path.join(bin_dir, script_name) | |||||
paths_to_remove = [exe_name] | |||||
if WINDOWS: | |||||
paths_to_remove.append(exe_name + '.exe') | |||||
paths_to_remove.append(exe_name + '.exe.manifest') | |||||
if is_gui: | |||||
paths_to_remove.append(exe_name + '-script.pyw') | |||||
else: | |||||
paths_to_remove.append(exe_name + '-script.py') | |||||
return paths_to_remove | |||||
def _unique(fn): | |||||
@functools.wraps(fn) | |||||
def unique(*args, **kw): | |||||
seen = set() | |||||
for item in fn(*args, **kw): | |||||
if item not in seen: | |||||
seen.add(item) | |||||
yield item | |||||
return unique | |||||
@_unique | |||||
def uninstallation_paths(dist): | |||||
""" | |||||
Yield all the uninstallation paths for dist based on RECORD-without-.pyc | |||||
Yield paths to all the files in RECORD. For each .py file in RECORD, add | |||||
the .pyc in the same directory. | |||||
UninstallPathSet.add() takes care of the __pycache__ .pyc. | |||||
""" | |||||
r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD'))) | |||||
for row in r: | |||||
path = os.path.join(dist.location, row[0]) | |||||
yield path | |||||
if path.endswith('.py'): | |||||
dn, fn = os.path.split(path) | |||||
base = fn[:-3] | |||||
path = os.path.join(dn, base + '.pyc') | |||||
yield path | |||||
def compact(paths): | |||||
"""Compact a path set to contain the minimal number of paths | |||||
necessary to contain all paths in the set. If /a/path/ and | |||||
/a/path/to/a/file.txt are both in the set, leave only the | |||||
shorter path.""" | |||||
sep = os.path.sep | |||||
short_paths = set() | |||||
for path in sorted(paths, key=len): | |||||
should_add = any( | |||||
path.startswith(shortpath.rstrip("*")) and | |||||
path[len(shortpath.rstrip("*").rstrip(sep))] == sep | |||||
for shortpath in short_paths | |||||
) | |||||
if not should_add: | |||||
short_paths.add(path) | |||||
return short_paths | |||||
def compress_for_output_listing(paths): | |||||
"""Returns a tuple of 2 sets of which paths to display to user | |||||
The first set contains paths that would be deleted. Files of a package | |||||
are not added and the top-level directory of the package has a '*' added | |||||
at the end - to signify that all it's contents are removed. | |||||
The second set contains files that would have been skipped in the above | |||||
folders. | |||||
""" | |||||
will_remove = list(paths) | |||||
will_skip = set() | |||||
# Determine folders and files | |||||
folders = set() | |||||
files = set() | |||||
for path in will_remove: | |||||
if path.endswith(".pyc"): | |||||
continue | |||||
if path.endswith("__init__.py") or ".dist-info" in path: | |||||
folders.add(os.path.dirname(path)) | |||||
files.add(path) | |||||
folders = compact(folders) | |||||
# This walks the tree using os.walk to not miss extra folders | |||||
# that might get added. | |||||
for folder in folders: | |||||
for dirpath, _, dirfiles in os.walk(folder): | |||||
for fname in dirfiles: | |||||
if fname.endswith(".pyc"): | |||||
continue | |||||
file_ = os.path.normcase(os.path.join(dirpath, fname)) | |||||
if os.path.isfile(file_) and file_ not in files: | |||||
# We are skipping this file. Add it to the set. | |||||
will_skip.add(file_) | |||||
will_remove = files | { | |||||
os.path.join(folder, "*") for folder in folders | |||||
} | |||||
return will_remove, will_skip | |||||
class UninstallPathSet(object): | |||||
"""A set of file paths to be removed in the uninstallation of a | |||||
requirement.""" | |||||
def __init__(self, dist): | |||||
self.paths = set() | |||||
self._refuse = set() | |||||
self.pth = {} | |||||
self.dist = dist | |||||
self.save_dir = TempDirectory(kind="uninstall") | |||||
self._moved_paths = [] | |||||
def _permitted(self, path): | |||||
""" | |||||
Return True if the given path is one we are permitted to | |||||
remove/modify, False otherwise. | |||||
""" | |||||
return is_local(path) | |||||
def add(self, path): | |||||
head, tail = os.path.split(path) | |||||
# we normalize the head to resolve parent directory symlinks, but not | |||||
# the tail, since we only want to uninstall symlinks, not their targets | |||||
path = os.path.join(normalize_path(head), os.path.normcase(tail)) | |||||
if not os.path.exists(path): | |||||
return | |||||
if self._permitted(path): | |||||
self.paths.add(path) | |||||
else: | |||||
self._refuse.add(path) | |||||
# __pycache__ files can show up after 'installed-files.txt' is created, | |||||
# due to imports | |||||
if os.path.splitext(path)[1] == '.py' and uses_pycache: | |||||
self.add(cache_from_source(path)) | |||||
def add_pth(self, pth_file, entry): | |||||
pth_file = normalize_path(pth_file) | |||||
if self._permitted(pth_file): | |||||
if pth_file not in self.pth: | |||||
self.pth[pth_file] = UninstallPthEntries(pth_file) | |||||
self.pth[pth_file].add(entry) | |||||
else: | |||||
self._refuse.add(pth_file) | |||||
def _stash(self, path): | |||||
return os.path.join( | |||||
self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep) | |||||
) | |||||
def remove(self, auto_confirm=False, verbose=False): | |||||
"""Remove paths in ``self.paths`` with confirmation (unless | |||||
``auto_confirm`` is True).""" | |||||
if not self.paths: | |||||
logger.info( | |||||
"Can't uninstall '%s'. No files were found to uninstall.", | |||||
self.dist.project_name, | |||||
) | |||||
return | |||||
dist_name_version = ( | |||||
self.dist.project_name + "-" + self.dist.version | |||||
) | |||||
logger.info('Uninstalling %s:', dist_name_version) | |||||
with indent_log(): | |||||
if auto_confirm or self._allowed_to_proceed(verbose): | |||||
self.save_dir.create() | |||||
for path in sorted(compact(self.paths)): | |||||
new_path = self._stash(path) | |||||
logger.debug('Removing file or directory %s', path) | |||||
self._moved_paths.append(path) | |||||
renames(path, new_path) | |||||
for pth in self.pth.values(): | |||||
pth.remove() | |||||
logger.info('Successfully uninstalled %s', dist_name_version) | |||||
def _allowed_to_proceed(self, verbose): | |||||
"""Display which files would be deleted and prompt for confirmation | |||||
""" | |||||
def _display(msg, paths): | |||||
if not paths: | |||||
return | |||||
logger.info(msg) | |||||
with indent_log(): | |||||
for path in sorted(compact(paths)): | |||||
logger.info(path) | |||||
if not verbose: | |||||
will_remove, will_skip = compress_for_output_listing(self.paths) | |||||
else: | |||||
# In verbose mode, display all the files that are going to be | |||||
# deleted. | |||||
will_remove = list(self.paths) | |||||
will_skip = set() | |||||
_display('Would remove:', will_remove) | |||||
_display('Would not remove (might be manually added):', will_skip) | |||||
_display('Would not remove (outside of prefix):', self._refuse) | |||||
return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' | |||||
def rollback(self): | |||||
"""Rollback the changes previously made by remove().""" | |||||
if self.save_dir.path is None: | |||||
logger.error( | |||||
"Can't roll back %s; was not uninstalled", | |||||
self.dist.project_name, | |||||
) | |||||
return False | |||||
logger.info('Rolling back uninstall of %s', self.dist.project_name) | |||||
for path in self._moved_paths: | |||||
tmp_path = self._stash(path) | |||||
logger.debug('Replacing %s', path) | |||||
renames(tmp_path, path) | |||||
for pth in self.pth.values(): | |||||
pth.rollback() | |||||
def commit(self): | |||||
"""Remove temporary save dir: rollback will no longer be possible.""" | |||||
self.save_dir.cleanup() | |||||
self._moved_paths = [] | |||||
@classmethod | |||||
def from_dist(cls, dist): | |||||
dist_path = normalize_path(dist.location) | |||||
if not dist_is_local(dist): | |||||
logger.info( | |||||
"Not uninstalling %s at %s, outside environment %s", | |||||
dist.key, | |||||
dist_path, | |||||
sys.prefix, | |||||
) | |||||
return cls(dist) | |||||
if dist_path in {p for p in {sysconfig.get_path("stdlib"), | |||||
sysconfig.get_path("platstdlib")} | |||||
if p}: | |||||
logger.info( | |||||
"Not uninstalling %s at %s, as it is in the standard library.", | |||||
dist.key, | |||||
dist_path, | |||||
) | |||||
return cls(dist) | |||||
paths_to_remove = cls(dist) | |||||
develop_egg_link = egg_link_path(dist) | |||||
develop_egg_link_egg_info = '{}.egg-info'.format( | |||||
pkg_resources.to_filename(dist.project_name)) | |||||
egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info) | |||||
# Special case for distutils installed package | |||||
distutils_egg_info = getattr(dist._provider, 'path', None) | |||||
# Uninstall cases order do matter as in the case of 2 installs of the | |||||
# same package, pip needs to uninstall the currently detected version | |||||
if (egg_info_exists and dist.egg_info.endswith('.egg-info') and | |||||
not dist.egg_info.endswith(develop_egg_link_egg_info)): | |||||
# if dist.egg_info.endswith(develop_egg_link_egg_info), we | |||||
# are in fact in the develop_egg_link case | |||||
paths_to_remove.add(dist.egg_info) | |||||
if dist.has_metadata('installed-files.txt'): | |||||
for installed_file in dist.get_metadata( | |||||
'installed-files.txt').splitlines(): | |||||
path = os.path.normpath( | |||||
os.path.join(dist.egg_info, installed_file) | |||||
) | |||||
paths_to_remove.add(path) | |||||
# FIXME: need a test for this elif block | |||||
# occurs with --single-version-externally-managed/--record outside | |||||
# of pip | |||||
elif dist.has_metadata('top_level.txt'): | |||||
if dist.has_metadata('namespace_packages.txt'): | |||||
namespaces = dist.get_metadata('namespace_packages.txt') | |||||
else: | |||||
namespaces = [] | |||||
for top_level_pkg in [ | |||||
p for p | |||||
in dist.get_metadata('top_level.txt').splitlines() | |||||
if p and p not in namespaces]: | |||||
path = os.path.join(dist.location, top_level_pkg) | |||||
paths_to_remove.add(path) | |||||
paths_to_remove.add(path + '.py') | |||||
paths_to_remove.add(path + '.pyc') | |||||
paths_to_remove.add(path + '.pyo') | |||||
elif distutils_egg_info: | |||||
raise UninstallationError( | |||||
"Cannot uninstall {!r}. It is a distutils installed project " | |||||
"and thus we cannot accurately determine which files belong " | |||||
"to it which would lead to only a partial uninstall.".format( | |||||
dist.project_name, | |||||
) | |||||
) | |||||
elif dist.location.endswith('.egg'): | |||||
# package installed by easy_install | |||||
# We cannot match on dist.egg_name because it can slightly vary | |||||
# i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg | |||||
paths_to_remove.add(dist.location) | |||||
easy_install_egg = os.path.split(dist.location)[1] | |||||
easy_install_pth = os.path.join(os.path.dirname(dist.location), | |||||
'easy-install.pth') | |||||
paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) | |||||
elif egg_info_exists and dist.egg_info.endswith('.dist-info'): | |||||
for path in uninstallation_paths(dist): | |||||
paths_to_remove.add(path) | |||||
elif develop_egg_link: | |||||
# develop egg | |||||
with open(develop_egg_link, 'r') as fh: | |||||
link_pointer = os.path.normcase(fh.readline().strip()) | |||||
assert (link_pointer == dist.location), ( | |||||
'Egg-link %s does not match installed location of %s ' | |||||
'(at %s)' % (link_pointer, dist.project_name, dist.location) | |||||
) | |||||
paths_to_remove.add(develop_egg_link) | |||||
easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), | |||||
'easy-install.pth') | |||||
paths_to_remove.add_pth(easy_install_pth, dist.location) | |||||
else: | |||||
logger.debug( | |||||
'Not sure how to uninstall: %s - Check: %s', | |||||
dist, dist.location, | |||||
) | |||||
# find distutils scripts= scripts | |||||
if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): | |||||
for script in dist.metadata_listdir('scripts'): | |||||
if dist_in_usersite(dist): | |||||
bin_dir = bin_user | |||||
else: | |||||
bin_dir = bin_py | |||||
paths_to_remove.add(os.path.join(bin_dir, script)) | |||||
if WINDOWS: | |||||
paths_to_remove.add(os.path.join(bin_dir, script) + '.bat') | |||||
# find console_scripts | |||||
_scripts_to_remove = [] | |||||
console_scripts = dist.get_entry_map(group='console_scripts') | |||||
for name in console_scripts.keys(): | |||||
_scripts_to_remove.extend(_script_names(dist, name, False)) | |||||
# find gui_scripts | |||||
gui_scripts = dist.get_entry_map(group='gui_scripts') | |||||
for name in gui_scripts.keys(): | |||||
_scripts_to_remove.extend(_script_names(dist, name, True)) | |||||
for s in _scripts_to_remove: | |||||
paths_to_remove.add(s) | |||||
return paths_to_remove | |||||
class UninstallPthEntries(object): | |||||
def __init__(self, pth_file): | |||||
if not os.path.isfile(pth_file): | |||||
raise UninstallationError( | |||||
"Cannot remove entries from nonexistent file %s" % pth_file | |||||
) | |||||
self.file = pth_file | |||||
self.entries = set() | |||||
self._saved_lines = None | |||||
def add(self, entry): | |||||
entry = os.path.normcase(entry) | |||||
# On Windows, os.path.normcase converts the entry to use | |||||
# backslashes. This is correct for entries that describe absolute | |||||
# paths outside of site-packages, but all the others use forward | |||||
# slashes. | |||||
if WINDOWS and not os.path.splitdrive(entry)[0]: | |||||
entry = entry.replace('\\', '/') | |||||
self.entries.add(entry) | |||||
def remove(self): | |||||
logger.debug('Removing pth entries from %s:', self.file) | |||||
with open(self.file, 'rb') as fh: | |||||
# windows uses '\r\n' with py3k, but uses '\n' with py2.x | |||||
lines = fh.readlines() | |||||
self._saved_lines = lines | |||||
if any(b'\r\n' in line for line in lines): | |||||
endline = '\r\n' | |||||
else: | |||||
endline = '\n' | |||||
# handle missing trailing newline | |||||
if lines and not lines[-1].endswith(endline.encode("utf-8")): | |||||
lines[-1] = lines[-1] + endline.encode("utf-8") | |||||
for entry in self.entries: | |||||
try: | |||||
logger.debug('Removing entry: %s', entry) | |||||
lines.remove((entry + endline).encode("utf-8")) | |||||
except ValueError: | |||||
pass | |||||
with open(self.file, 'wb') as fh: | |||||
fh.writelines(lines) | |||||
def rollback(self): | |||||
if self._saved_lines is None: | |||||
logger.error( | |||||
'Cannot roll back changes to %s, none were made', self.file | |||||
) | |||||
return False | |||||
logger.debug('Rolling %s back to previous state', self.file) | |||||
with open(self.file, 'wb') as fh: | |||||
fh.writelines(self._saved_lines) | |||||
return True |
"""Dependency Resolution | |||||
The dependency resolution in pip is performed as follows: | |||||
for top-level requirements: | |||||
a. only one spec allowed per project, regardless of conflicts or not. | |||||
otherwise a "double requirement" exception is raised | |||||
b. they override sub-dependency requirements. | |||||
for sub-dependencies | |||||
a. "first found, wins" (where the order is breadth first) | |||||
""" | |||||
import logging | |||||
from collections import defaultdict | |||||
from itertools import chain | |||||
from pip._internal.exceptions import ( | |||||
BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, | |||||
UnsupportedPythonVersion, | |||||
) | |||||
from pip._internal.req.req_install import InstallRequirement | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.utils.misc import dist_in_usersite, ensure_dir | |||||
from pip._internal.utils.packaging import check_dist_requires_python | |||||
logger = logging.getLogger(__name__) | |||||
class Resolver(object): | |||||
"""Resolves which packages need to be installed/uninstalled to perform \ | |||||
the requested operation without breaking the requirements of any package. | |||||
""" | |||||
_allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"} | |||||
def __init__(self, preparer, session, finder, wheel_cache, use_user_site, | |||||
ignore_dependencies, ignore_installed, ignore_requires_python, | |||||
force_reinstall, isolated, upgrade_strategy): | |||||
super(Resolver, self).__init__() | |||||
assert upgrade_strategy in self._allowed_strategies | |||||
self.preparer = preparer | |||||
self.finder = finder | |||||
self.session = session | |||||
# NOTE: This would eventually be replaced with a cache that can give | |||||
# information about both sdist and wheels transparently. | |||||
self.wheel_cache = wheel_cache | |||||
self.require_hashes = None # This is set in resolve | |||||
self.upgrade_strategy = upgrade_strategy | |||||
self.force_reinstall = force_reinstall | |||||
self.isolated = isolated | |||||
self.ignore_dependencies = ignore_dependencies | |||||
self.ignore_installed = ignore_installed | |||||
self.ignore_requires_python = ignore_requires_python | |||||
self.use_user_site = use_user_site | |||||
self._discovered_dependencies = defaultdict(list) | |||||
def resolve(self, requirement_set): | |||||
"""Resolve what operations need to be done | |||||
As a side-effect of this method, the packages (and their dependencies) | |||||
are downloaded, unpacked and prepared for installation. This | |||||
preparation is done by ``pip.operations.prepare``. | |||||
Once PyPI has static dependency metadata available, it would be | |||||
possible to move the preparation to become a step separated from | |||||
dependency resolution. | |||||
""" | |||||
# make the wheelhouse | |||||
if self.preparer.wheel_download_dir: | |||||
ensure_dir(self.preparer.wheel_download_dir) | |||||
# If any top-level requirement has a hash specified, enter | |||||
# hash-checking mode, which requires hashes from all. | |||||
root_reqs = ( | |||||
requirement_set.unnamed_requirements + | |||||
list(requirement_set.requirements.values()) | |||||
) | |||||
self.require_hashes = ( | |||||
requirement_set.require_hashes or | |||||
any(req.has_hash_options for req in root_reqs) | |||||
) | |||||
# Display where finder is looking for packages | |||||
locations = self.finder.get_formatted_locations() | |||||
if locations: | |||||
logger.info(locations) | |||||
# Actually prepare the files, and collect any exceptions. Most hash | |||||
# exceptions cannot be checked ahead of time, because | |||||
# req.populate_link() needs to be called before we can make decisions | |||||
# based on link type. | |||||
discovered_reqs = [] | |||||
hash_errors = HashErrors() | |||||
for req in chain(root_reqs, discovered_reqs): | |||||
try: | |||||
discovered_reqs.extend( | |||||
self._resolve_one(requirement_set, req) | |||||
) | |||||
except HashError as exc: | |||||
exc.req = req | |||||
hash_errors.append(exc) | |||||
if hash_errors: | |||||
raise hash_errors | |||||
def _is_upgrade_allowed(self, req): | |||||
if self.upgrade_strategy == "to-satisfy-only": | |||||
return False | |||||
elif self.upgrade_strategy == "eager": | |||||
return True | |||||
else: | |||||
assert self.upgrade_strategy == "only-if-needed" | |||||
return req.is_direct | |||||
def _set_req_to_reinstall(self, req): | |||||
""" | |||||
Set a requirement to be installed. | |||||
""" | |||||
# Don't uninstall the conflict if doing a user install and the | |||||
# conflict is not a user install. | |||||
if not self.use_user_site or dist_in_usersite(req.satisfied_by): | |||||
req.conflicts_with = req.satisfied_by | |||||
req.satisfied_by = None | |||||
# XXX: Stop passing requirement_set for options | |||||
def _check_skip_installed(self, req_to_install): | |||||
"""Check if req_to_install should be skipped. | |||||
This will check if the req is installed, and whether we should upgrade | |||||
or reinstall it, taking into account all the relevant user options. | |||||
After calling this req_to_install will only have satisfied_by set to | |||||
None if the req_to_install is to be upgraded/reinstalled etc. Any | |||||
other value will be a dist recording the current thing installed that | |||||
satisfies the requirement. | |||||
Note that for vcs urls and the like we can't assess skipping in this | |||||
routine - we simply identify that we need to pull the thing down, | |||||
then later on it is pulled down and introspected to assess upgrade/ | |||||
reinstalls etc. | |||||
:return: A text reason for why it was skipped, or None. | |||||
""" | |||||
if self.ignore_installed: | |||||
return None | |||||
req_to_install.check_if_exists(self.use_user_site) | |||||
if not req_to_install.satisfied_by: | |||||
return None | |||||
if self.force_reinstall: | |||||
self._set_req_to_reinstall(req_to_install) | |||||
return None | |||||
if not self._is_upgrade_allowed(req_to_install): | |||||
if self.upgrade_strategy == "only-if-needed": | |||||
return 'not upgraded as not directly required' | |||||
return 'already satisfied' | |||||
# Check for the possibility of an upgrade. For link-based | |||||
# requirements we have to pull the tree down and inspect to assess | |||||
# the version #, so it's handled way down. | |||||
if not req_to_install.link: | |||||
try: | |||||
self.finder.find_requirement(req_to_install, upgrade=True) | |||||
except BestVersionAlreadyInstalled: | |||||
# Then the best version is installed. | |||||
return 'already up-to-date' | |||||
except DistributionNotFound: | |||||
# No distribution found, so we squash the error. It will | |||||
# be raised later when we re-try later to do the install. | |||||
# Why don't we just raise here? | |||||
pass | |||||
self._set_req_to_reinstall(req_to_install) | |||||
return None | |||||
def _get_abstract_dist_for(self, req): | |||||
"""Takes a InstallRequirement and returns a single AbstractDist \ | |||||
representing a prepared variant of the same. | |||||
""" | |||||
assert self.require_hashes is not None, ( | |||||
"require_hashes should have been set in Resolver.resolve()" | |||||
) | |||||
if req.editable: | |||||
return self.preparer.prepare_editable_requirement( | |||||
req, self.require_hashes, self.use_user_site, self.finder, | |||||
) | |||||
# satisfied_by is only evaluated by calling _check_skip_installed, | |||||
# so it must be None here. | |||||
assert req.satisfied_by is None | |||||
skip_reason = self._check_skip_installed(req) | |||||
if req.satisfied_by: | |||||
return self.preparer.prepare_installed_requirement( | |||||
req, self.require_hashes, skip_reason | |||||
) | |||||
upgrade_allowed = self._is_upgrade_allowed(req) | |||||
abstract_dist = self.preparer.prepare_linked_requirement( | |||||
req, self.session, self.finder, upgrade_allowed, | |||||
self.require_hashes | |||||
) | |||||
# NOTE | |||||
# The following portion is for determining if a certain package is | |||||
# going to be re-installed/upgraded or not and reporting to the user. | |||||
# This should probably get cleaned up in a future refactor. | |||||
# req.req is only avail after unpack for URL | |||||
# pkgs repeat check_if_exists to uninstall-on-upgrade | |||||
# (#14) | |||||
if not self.ignore_installed: | |||||
req.check_if_exists(self.use_user_site) | |||||
if req.satisfied_by: | |||||
should_modify = ( | |||||
self.upgrade_strategy != "to-satisfy-only" or | |||||
self.force_reinstall or | |||||
self.ignore_installed or | |||||
req.link.scheme == 'file' | |||||
) | |||||
if should_modify: | |||||
self._set_req_to_reinstall(req) | |||||
else: | |||||
logger.info( | |||||
'Requirement already satisfied (use --upgrade to upgrade):' | |||||
' %s', req, | |||||
) | |||||
return abstract_dist | |||||
def _resolve_one(self, requirement_set, req_to_install): | |||||
"""Prepare a single requirements file. | |||||
:return: A list of additional InstallRequirements to also install. | |||||
""" | |||||
# Tell user what we are doing for this requirement: | |||||
# obtain (editable), skipping, processing (local url), collecting | |||||
# (remote url or package name) | |||||
if req_to_install.constraint or req_to_install.prepared: | |||||
return [] | |||||
req_to_install.prepared = True | |||||
# register tmp src for cleanup in case something goes wrong | |||||
requirement_set.reqs_to_cleanup.append(req_to_install) | |||||
abstract_dist = self._get_abstract_dist_for(req_to_install) | |||||
# Parse and return dependencies | |||||
dist = abstract_dist.dist(self.finder) | |||||
try: | |||||
check_dist_requires_python(dist) | |||||
except UnsupportedPythonVersion as err: | |||||
if self.ignore_requires_python: | |||||
logger.warning(err.args[0]) | |||||
else: | |||||
raise | |||||
more_reqs = [] | |||||
def add_req(subreq, extras_requested): | |||||
sub_install_req = InstallRequirement.from_req( | |||||
str(subreq), | |||||
req_to_install, | |||||
isolated=self.isolated, | |||||
wheel_cache=self.wheel_cache, | |||||
) | |||||
parent_req_name = req_to_install.name | |||||
to_scan_again, add_to_parent = requirement_set.add_requirement( | |||||
sub_install_req, | |||||
parent_req_name=parent_req_name, | |||||
extras_requested=extras_requested, | |||||
) | |||||
if parent_req_name and add_to_parent: | |||||
self._discovered_dependencies[parent_req_name].append( | |||||
add_to_parent | |||||
) | |||||
more_reqs.extend(to_scan_again) | |||||
with indent_log(): | |||||
# We add req_to_install before its dependencies, so that we | |||||
# can refer to it when adding dependencies. | |||||
if not requirement_set.has_requirement(req_to_install.name): | |||||
# 'unnamed' requirements will get added here | |||||
req_to_install.is_direct = True | |||||
requirement_set.add_requirement( | |||||
req_to_install, parent_req_name=None, | |||||
) | |||||
if not self.ignore_dependencies: | |||||
if req_to_install.extras: | |||||
logger.debug( | |||||
"Installing extra requirements: %r", | |||||
','.join(req_to_install.extras), | |||||
) | |||||
missing_requested = sorted( | |||||
set(req_to_install.extras) - set(dist.extras) | |||||
) | |||||
for missing in missing_requested: | |||||
logger.warning( | |||||
'%s does not provide the extra \'%s\'', | |||||
dist, missing | |||||
) | |||||
available_requested = sorted( | |||||
set(dist.extras) & set(req_to_install.extras) | |||||
) | |||||
for subreq in dist.requires(available_requested): | |||||
add_req(subreq, extras_requested=available_requested) | |||||
if not req_to_install.editable and not req_to_install.satisfied_by: | |||||
# XXX: --no-install leads this to report 'Successfully | |||||
# downloaded' for only non-editable reqs, even though we took | |||||
# action on them. | |||||
requirement_set.successfully_downloaded.append(req_to_install) | |||||
return more_reqs | |||||
def get_installation_order(self, req_set): | |||||
"""Create the installation order. | |||||
The installation order is topological - requirements are installed | |||||
before the requiring thing. We break cycles at an arbitrary point, | |||||
and make no other guarantees. | |||||
""" | |||||
# The current implementation, which we may change at any point | |||||
# installs the user specified things in the order given, except when | |||||
# dependencies must come earlier to achieve topological order. | |||||
order = [] | |||||
ordered_reqs = set() | |||||
def schedule(req): | |||||
if req.satisfied_by or req in ordered_reqs: | |||||
return | |||||
if req.constraint: | |||||
return | |||||
ordered_reqs.add(req) | |||||
for dep in self._discovered_dependencies[req.name]: | |||||
schedule(dep) | |||||
order.append(req) | |||||
for install_req in req_set.requirements.values(): | |||||
schedule(install_req) | |||||
return order |
from __future__ import absolute_import | |||||
SUCCESS = 0 | |||||
ERROR = 1 | |||||
UNKNOWN_ERROR = 2 | |||||
VIRTUALENV_NOT_FOUND = 3 | |||||
PREVIOUS_BUILD_DIR_ERROR = 4 | |||||
NO_MATCHES_FOUND = 23 |
""" | |||||
This code was taken from https://github.com/ActiveState/appdirs and modified | |||||
to suit our purposes. | |||||
""" | |||||
from __future__ import absolute_import | |||||
import os | |||||
import sys | |||||
from pip._vendor.six import PY2, text_type | |||||
from pip._internal.compat import WINDOWS, expanduser | |||||
def user_cache_dir(appname): | |||||
r""" | |||||
Return full path to the user-specific cache dir for this application. | |||||
"appname" is the name of application. | |||||
Typical user cache directories are: | |||||
macOS: ~/Library/Caches/<AppName> | |||||
Unix: ~/.cache/<AppName> (XDG default) | |||||
Windows: C:\Users\<username>\AppData\Local\<AppName>\Cache | |||||
On Windows the only suggestion in the MSDN docs is that local settings go | |||||
in the `CSIDL_LOCAL_APPDATA` directory. This is identical to the | |||||
non-roaming app data dir (the default returned by `user_data_dir`). Apps | |||||
typically put cache data somewhere *under* the given dir here. Some | |||||
examples: | |||||
...\Mozilla\Firefox\Profiles\<ProfileName>\Cache | |||||
...\Acme\SuperApp\Cache\1.0 | |||||
OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. | |||||
""" | |||||
if WINDOWS: | |||||
# Get the base path | |||||
path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) | |||||
# When using Python 2, return paths as bytes on Windows like we do on | |||||
# other operating systems. See helper function docs for more details. | |||||
if PY2 and isinstance(path, text_type): | |||||
path = _win_path_to_bytes(path) | |||||
# Add our app name and Cache directory to it | |||||
path = os.path.join(path, appname, "Cache") | |||||
elif sys.platform == "darwin": | |||||
# Get the base path | |||||
path = expanduser("~/Library/Caches") | |||||
# Add our app name to it | |||||
path = os.path.join(path, appname) | |||||
else: | |||||
# Get the base path | |||||
path = os.getenv("XDG_CACHE_HOME", expanduser("~/.cache")) | |||||
# Add our app name to it | |||||
path = os.path.join(path, appname) | |||||
return path | |||||
def user_data_dir(appname, roaming=False): | |||||
r""" | |||||
Return full path to the user-specific data dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"roaming" (boolean, default False) can be set True to use the Windows | |||||
roaming appdata directory. That means that for users on a Windows | |||||
network setup for roaming profiles, this user data will be | |||||
sync'd on login. See | |||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> | |||||
for a discussion of issues. | |||||
Typical user data directories are: | |||||
macOS: ~/Library/Application Support/<AppName> | |||||
if it exists, else ~/.config/<AppName> | |||||
Unix: ~/.local/share/<AppName> # or in | |||||
$XDG_DATA_HOME, if defined | |||||
Win XP (not roaming): C:\Documents and Settings\<username>\ ... | |||||
...Application Data\<AppName> | |||||
Win XP (roaming): C:\Documents and Settings\<username>\Local ... | |||||
...Settings\Application Data\<AppName> | |||||
Win 7 (not roaming): C:\\Users\<username>\AppData\Local\<AppName> | |||||
Win 7 (roaming): C:\\Users\<username>\AppData\Roaming\<AppName> | |||||
For Unix, we follow the XDG spec and support $XDG_DATA_HOME. | |||||
That means, by default "~/.local/share/<AppName>". | |||||
""" | |||||
if WINDOWS: | |||||
const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" | |||||
path = os.path.join(os.path.normpath(_get_win_folder(const)), appname) | |||||
elif sys.platform == "darwin": | |||||
path = os.path.join( | |||||
expanduser('~/Library/Application Support/'), | |||||
appname, | |||||
) if os.path.isdir(os.path.join( | |||||
expanduser('~/Library/Application Support/'), | |||||
appname, | |||||
) | |||||
) else os.path.join( | |||||
expanduser('~/.config/'), | |||||
appname, | |||||
) | |||||
else: | |||||
path = os.path.join( | |||||
os.getenv('XDG_DATA_HOME', expanduser("~/.local/share")), | |||||
appname, | |||||
) | |||||
return path | |||||
def user_config_dir(appname, roaming=True): | |||||
"""Return full path to the user-specific config dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"roaming" (boolean, default True) can be set False to not use the | |||||
Windows roaming appdata directory. That means that for users on a | |||||
Windows network setup for roaming profiles, this user data will be | |||||
sync'd on login. See | |||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> | |||||
for a discussion of issues. | |||||
Typical user data directories are: | |||||
macOS: same as user_data_dir | |||||
Unix: ~/.config/<AppName> | |||||
Win *: same as user_data_dir | |||||
For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. | |||||
That means, by default "~/.config/<AppName>". | |||||
""" | |||||
if WINDOWS: | |||||
path = user_data_dir(appname, roaming=roaming) | |||||
elif sys.platform == "darwin": | |||||
path = user_data_dir(appname) | |||||
else: | |||||
path = os.getenv('XDG_CONFIG_HOME', expanduser("~/.config")) | |||||
path = os.path.join(path, appname) | |||||
return path | |||||
# for the discussion regarding site_config_dirs locations | |||||
# see <https://github.com/pypa/pip/issues/1733> | |||||
def site_config_dirs(appname): | |||||
r"""Return a list of potential user-shared config dirs for this application. | |||||
"appname" is the name of application. | |||||
Typical user config directories are: | |||||
macOS: /Library/Application Support/<AppName>/ | |||||
Unix: /etc or $XDG_CONFIG_DIRS[i]/<AppName>/ for each value in | |||||
$XDG_CONFIG_DIRS | |||||
Win XP: C:\Documents and Settings\All Users\Application ... | |||||
...Data\<AppName>\ | |||||
Vista: (Fail! "C:\ProgramData" is a hidden *system* directory | |||||
on Vista.) | |||||
Win 7: Hidden, but writeable on Win 7: | |||||
C:\ProgramData\<AppName>\ | |||||
""" | |||||
if WINDOWS: | |||||
path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) | |||||
pathlist = [os.path.join(path, appname)] | |||||
elif sys.platform == 'darwin': | |||||
pathlist = [os.path.join('/Library/Application Support', appname)] | |||||
else: | |||||
# try looking in $XDG_CONFIG_DIRS | |||||
xdg_config_dirs = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') | |||||
if xdg_config_dirs: | |||||
pathlist = [ | |||||
os.path.join(expanduser(x), appname) | |||||
for x in xdg_config_dirs.split(os.pathsep) | |||||
] | |||||
else: | |||||
pathlist = [] | |||||
# always look in /etc directly as well | |||||
pathlist.append('/etc') | |||||
return pathlist | |||||
# -- Windows support functions -- | |||||
def _get_win_folder_from_registry(csidl_name): | |||||
""" | |||||
This is a fallback technique at best. I'm not sure if using the | |||||
registry for this guarantees us the correct answer for all CSIDL_* | |||||
names. | |||||
""" | |||||
import _winreg | |||||
shell_folder_name = { | |||||
"CSIDL_APPDATA": "AppData", | |||||
"CSIDL_COMMON_APPDATA": "Common AppData", | |||||
"CSIDL_LOCAL_APPDATA": "Local AppData", | |||||
}[csidl_name] | |||||
key = _winreg.OpenKey( | |||||
_winreg.HKEY_CURRENT_USER, | |||||
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" | |||||
) | |||||
directory, _type = _winreg.QueryValueEx(key, shell_folder_name) | |||||
return directory | |||||
def _get_win_folder_with_ctypes(csidl_name): | |||||
csidl_const = { | |||||
"CSIDL_APPDATA": 26, | |||||
"CSIDL_COMMON_APPDATA": 35, | |||||
"CSIDL_LOCAL_APPDATA": 28, | |||||
}[csidl_name] | |||||
buf = ctypes.create_unicode_buffer(1024) | |||||
ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) | |||||
# Downgrade to short path name if have highbit chars. See | |||||
# <http://bugs.activestate.com/show_bug.cgi?id=85099>. | |||||
has_high_char = False | |||||
for c in buf: | |||||
if ord(c) > 255: | |||||
has_high_char = True | |||||
break | |||||
if has_high_char: | |||||
buf2 = ctypes.create_unicode_buffer(1024) | |||||
if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): | |||||
buf = buf2 | |||||
return buf.value | |||||
if WINDOWS: | |||||
try: | |||||
import ctypes | |||||
_get_win_folder = _get_win_folder_with_ctypes | |||||
except ImportError: | |||||
_get_win_folder = _get_win_folder_from_registry | |||||
def _win_path_to_bytes(path): | |||||
"""Encode Windows paths to bytes. Only used on Python 2. | |||||
Motivation is to be consistent with other operating systems where paths | |||||
are also returned as bytes. This avoids problems mixing bytes and Unicode | |||||
elsewhere in the codebase. For more details and discussion see | |||||
<https://github.com/pypa/pip/issues/3463>. | |||||
If encoding using ASCII and MBCS fails, return the original Unicode path. | |||||
""" | |||||
for encoding in ('ASCII', 'MBCS'): | |||||
try: | |||||
return path.encode(encoding) | |||||
except (UnicodeEncodeError, LookupError): | |||||
pass | |||||
return path |
""" | |||||
A module that implements tooling to enable easy warnings about deprecations. | |||||
""" | |||||
from __future__ import absolute_import | |||||
import logging | |||||
import warnings | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Any | |||||
class PipDeprecationWarning(Warning): | |||||
pass | |||||
class Pending(object): | |||||
pass | |||||
class RemovedInPip11Warning(PipDeprecationWarning): | |||||
pass | |||||
class RemovedInPip12Warning(PipDeprecationWarning, Pending): | |||||
pass | |||||
# Warnings <-> Logging Integration | |||||
_warnings_showwarning = None # type: Any | |||||
def _showwarning(message, category, filename, lineno, file=None, line=None): | |||||
if file is not None: | |||||
if _warnings_showwarning is not None: | |||||
_warnings_showwarning( | |||||
message, category, filename, lineno, file, line, | |||||
) | |||||
else: | |||||
if issubclass(category, PipDeprecationWarning): | |||||
# We use a specially named logger which will handle all of the | |||||
# deprecation messages for pip. | |||||
logger = logging.getLogger("pip._internal.deprecations") | |||||
# This is purposely using the % formatter here instead of letting | |||||
# the logging module handle the interpolation. This is because we | |||||
# want it to appear as if someone typed this entire message out. | |||||
log_message = "DEPRECATION: %s" % message | |||||
# PipDeprecationWarnings that are Pending still have at least 2 | |||||
# versions to go until they are removed so they can just be | |||||
# warnings. Otherwise, they will be removed in the very next | |||||
# version of pip. We want these to be more obvious so we use the | |||||
# ERROR logging level. | |||||
if issubclass(category, Pending): | |||||
logger.warning(log_message) | |||||
else: | |||||
logger.error(log_message) | |||||
else: | |||||
_warnings_showwarning( | |||||
message, category, filename, lineno, file, line, | |||||
) | |||||
def install_warning_logger(): | |||||
# Enable our Deprecation Warnings | |||||
warnings.simplefilter("default", PipDeprecationWarning, append=True) | |||||
global _warnings_showwarning | |||||
if _warnings_showwarning is None: | |||||
_warnings_showwarning = warnings.showwarning | |||||
warnings.showwarning = _showwarning |
import codecs | |||||
import locale | |||||
import re | |||||
import sys | |||||
BOMS = [ | |||||
(codecs.BOM_UTF8, 'utf8'), | |||||
(codecs.BOM_UTF16, 'utf16'), | |||||
(codecs.BOM_UTF16_BE, 'utf16-be'), | |||||
(codecs.BOM_UTF16_LE, 'utf16-le'), | |||||
(codecs.BOM_UTF32, 'utf32'), | |||||
(codecs.BOM_UTF32_BE, 'utf32-be'), | |||||
(codecs.BOM_UTF32_LE, 'utf32-le'), | |||||
] | |||||
ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') | |||||
def auto_decode(data): | |||||
"""Check a bytes string for a BOM to correctly detect the encoding | |||||
Fallback to locale.getpreferredencoding(False) like open() on Python3""" | |||||
for bom, encoding in BOMS: | |||||
if data.startswith(bom): | |||||
return data[len(bom):].decode(encoding) | |||||
# Lets check the first two lines as in PEP263 | |||||
for line in data.split(b'\n')[:2]: | |||||
if line[0:1] == b'#' and ENCODING_RE.search(line): | |||||
encoding = ENCODING_RE.search(line).groups()[0].decode('ascii') | |||||
return data.decode(encoding) | |||||
return data.decode( | |||||
locale.getpreferredencoding(False) or sys.getdefaultencoding(), | |||||
) |
import os | |||||
import os.path | |||||
from pip._internal.compat import get_path_uid | |||||
def check_path_owner(path): | |||||
# If we don't have a way to check the effective uid of this process, then | |||||
# we'll just assume that we own the directory. | |||||
if not hasattr(os, "geteuid"): | |||||
return True | |||||
previous = None | |||||
while path != previous: | |||||
if os.path.lexists(path): | |||||
# Check if path is writable by current user. | |||||
if os.geteuid() == 0: | |||||
# Special handling for root user in order to handle properly | |||||
# cases where users use sudo without -H flag. | |||||
try: | |||||
path_uid = get_path_uid(path) | |||||
except OSError: | |||||
return False | |||||
return path_uid == 0 | |||||
else: | |||||
return os.access(path, os.W_OK) | |||||
else: | |||||
previous, path = path, os.path.dirname(path) |
from __future__ import absolute_import | |||||
import ctypes | |||||
import re | |||||
import warnings | |||||
def glibc_version_string(): | |||||
"Returns glibc version string, or None if not using glibc." | |||||
# ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen | |||||
# manpage says, "If filename is NULL, then the returned handle is for the | |||||
# main program". This way we can let the linker do the work to figure out | |||||
# which libc our process is actually using. | |||||
process_namespace = ctypes.CDLL(None) | |||||
try: | |||||
gnu_get_libc_version = process_namespace.gnu_get_libc_version | |||||
except AttributeError: | |||||
# Symbol doesn't exist -> therefore, we are not linked to | |||||
# glibc. | |||||
return None | |||||
# Call gnu_get_libc_version, which returns a string like "2.5" | |||||
gnu_get_libc_version.restype = ctypes.c_char_p | |||||
version_str = gnu_get_libc_version() | |||||
# py2 / py3 compatibility: | |||||
if not isinstance(version_str, str): | |||||
version_str = version_str.decode("ascii") | |||||
return version_str | |||||
# Separated out from have_compatible_glibc for easier unit testing | |||||
def check_glibc_version(version_str, required_major, minimum_minor): | |||||
# Parse string and check against requested version. | |||||
# | |||||
# We use a regexp instead of str.split because we want to discard any | |||||
# random junk that might come after the minor version -- this might happen | |||||
# in patched/forked versions of glibc (e.g. Linaro's version of glibc | |||||
# uses version strings like "2.20-2014.11"). See gh-3588. | |||||
m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) | |||||
if not m: | |||||
warnings.warn("Expected glibc version with 2 components major.minor," | |||||
" got: %s" % version_str, RuntimeWarning) | |||||
return False | |||||
return (int(m.group("major")) == required_major and | |||||
int(m.group("minor")) >= minimum_minor) | |||||
def have_compatible_glibc(required_major, minimum_minor): | |||||
version_str = glibc_version_string() | |||||
if version_str is None: | |||||
return False | |||||
return check_glibc_version(version_str, required_major, minimum_minor) | |||||
# platform.libc_ver regularly returns completely nonsensical glibc | |||||
# versions. E.g. on my computer, platform says: | |||||
# | |||||
# ~$ python2.7 -c 'import platform; print(platform.libc_ver())' | |||||
# ('glibc', '2.7') | |||||
# ~$ python3.5 -c 'import platform; print(platform.libc_ver())' | |||||
# ('glibc', '2.9') | |||||
# | |||||
# But the truth is: | |||||
# | |||||
# ~$ ldd --version | |||||
# ldd (Debian GLIBC 2.22-11) 2.22 | |||||
# | |||||
# This is unfortunate, because it means that the linehaul data on libc | |||||
# versions that was generated by pip 8.1.2 and earlier is useless and | |||||
# misleading. Solution: instead of using platform, use our code that actually | |||||
# works. | |||||
def libc_ver(): | |||||
"""Try to determine the glibc version | |||||
Returns a tuple of strings (lib, version) which default to empty strings | |||||
in case the lookup fails. | |||||
""" | |||||
glibc_version = glibc_version_string() | |||||
if glibc_version is None: | |||||
return ("", "") | |||||
else: | |||||
return ("glibc", glibc_version) |
from __future__ import absolute_import | |||||
import hashlib | |||||
from pip._vendor.six import iteritems, iterkeys, itervalues | |||||
from pip._internal.exceptions import ( | |||||
HashMismatch, HashMissing, InstallationError, | |||||
) | |||||
from pip._internal.utils.misc import read_chunks | |||||
# The recommended hash algo of the moment. Change this whenever the state of | |||||
# the art changes; it won't hurt backward compatibility. | |||||
FAVORITE_HASH = 'sha256' | |||||
# Names of hashlib algorithms allowed by the --hash option and ``pip hash`` | |||||
# Currently, those are the ones at least as collision-resistant as sha256. | |||||
STRONG_HASHES = ['sha256', 'sha384', 'sha512'] | |||||
class Hashes(object): | |||||
"""A wrapper that builds multiple hashes at once and checks them against | |||||
known-good values | |||||
""" | |||||
def __init__(self, hashes=None): | |||||
""" | |||||
:param hashes: A dict of algorithm names pointing to lists of allowed | |||||
hex digests | |||||
""" | |||||
self._allowed = {} if hashes is None else hashes | |||||
def check_against_chunks(self, chunks): | |||||
"""Check good hashes against ones built from iterable of chunks of | |||||
data. | |||||
Raise HashMismatch if none match. | |||||
""" | |||||
gots = {} | |||||
for hash_name in iterkeys(self._allowed): | |||||
try: | |||||
gots[hash_name] = hashlib.new(hash_name) | |||||
except (ValueError, TypeError): | |||||
raise InstallationError('Unknown hash name: %s' % hash_name) | |||||
for chunk in chunks: | |||||
for hash in itervalues(gots): | |||||
hash.update(chunk) | |||||
for hash_name, got in iteritems(gots): | |||||
if got.hexdigest() in self._allowed[hash_name]: | |||||
return | |||||
self._raise(gots) | |||||
def _raise(self, gots): | |||||
raise HashMismatch(self._allowed, gots) | |||||
def check_against_file(self, file): | |||||
"""Check good hashes against a file-like object | |||||
Raise HashMismatch if none match. | |||||
""" | |||||
return self.check_against_chunks(read_chunks(file)) | |||||
def check_against_path(self, path): | |||||
with open(path, 'rb') as file: | |||||
return self.check_against_file(file) | |||||
def __nonzero__(self): | |||||
"""Return whether I know any known-good hashes.""" | |||||
return bool(self._allowed) | |||||
def __bool__(self): | |||||
return self.__nonzero__() | |||||
class MissingHashes(Hashes): | |||||
"""A workalike for Hashes used when we're missing a hash for a requirement | |||||
It computes the actual hash of the requirement and raises a HashMissing | |||||
exception showing it to the user. | |||||
""" | |||||
def __init__(self): | |||||
"""Don't offer the ``hashes`` kwarg.""" | |||||
# Pass our favorite hash in to generate a "gotten hash". With the | |||||
# empty list, it will never match, so an error will always raise. | |||||
super(MissingHashes, self).__init__(hashes={FAVORITE_HASH: []}) | |||||
def _raise(self, gots): | |||||
raise HashMissing(gots[FAVORITE_HASH].hexdigest()) |
from __future__ import absolute_import | |||||
import contextlib | |||||
import logging | |||||
import logging.handlers | |||||
import os | |||||
from pip._internal.compat import WINDOWS | |||||
from pip._internal.utils.misc import ensure_dir | |||||
try: | |||||
import threading | |||||
except ImportError: | |||||
import dummy_threading as threading # type: ignore | |||||
try: | |||||
from pip._vendor import colorama | |||||
# Lots of different errors can come from this, including SystemError and | |||||
# ImportError. | |||||
except Exception: | |||||
colorama = None | |||||
_log_state = threading.local() | |||||
_log_state.indentation = 0 | |||||
@contextlib.contextmanager | |||||
def indent_log(num=2): | |||||
""" | |||||
A context manager which will cause the log output to be indented for any | |||||
log messages emitted inside it. | |||||
""" | |||||
_log_state.indentation += num | |||||
try: | |||||
yield | |||||
finally: | |||||
_log_state.indentation -= num | |||||
def get_indentation(): | |||||
return getattr(_log_state, 'indentation', 0) | |||||
class IndentingFormatter(logging.Formatter): | |||||
def format(self, record): | |||||
""" | |||||
Calls the standard formatter, but will indent all of the log messages | |||||
by our current indentation level. | |||||
""" | |||||
formatted = logging.Formatter.format(self, record) | |||||
formatted = "".join([ | |||||
(" " * get_indentation()) + line | |||||
for line in formatted.splitlines(True) | |||||
]) | |||||
return formatted | |||||
def _color_wrap(*colors): | |||||
def wrapped(inp): | |||||
return "".join(list(colors) + [inp, colorama.Style.RESET_ALL]) | |||||
return wrapped | |||||
class ColorizedStreamHandler(logging.StreamHandler): | |||||
# Don't build up a list of colors if we don't have colorama | |||||
if colorama: | |||||
COLORS = [ | |||||
# This needs to be in order from highest logging level to lowest. | |||||
(logging.ERROR, _color_wrap(colorama.Fore.RED)), | |||||
(logging.WARNING, _color_wrap(colorama.Fore.YELLOW)), | |||||
] | |||||
else: | |||||
COLORS = [] | |||||
def __init__(self, stream=None, no_color=None): | |||||
logging.StreamHandler.__init__(self, stream) | |||||
self._no_color = no_color | |||||
if WINDOWS and colorama: | |||||
self.stream = colorama.AnsiToWin32(self.stream) | |||||
def should_color(self): | |||||
# Don't colorize things if we do not have colorama or if told not to | |||||
if not colorama or self._no_color: | |||||
return False | |||||
real_stream = ( | |||||
self.stream if not isinstance(self.stream, colorama.AnsiToWin32) | |||||
else self.stream.wrapped | |||||
) | |||||
# If the stream is a tty we should color it | |||||
if hasattr(real_stream, "isatty") and real_stream.isatty(): | |||||
return True | |||||
# If we have an ASNI term we should color it | |||||
if os.environ.get("TERM") == "ANSI": | |||||
return True | |||||
# If anything else we should not color it | |||||
return False | |||||
def format(self, record): | |||||
msg = logging.StreamHandler.format(self, record) | |||||
if self.should_color(): | |||||
for level, color in self.COLORS: | |||||
if record.levelno >= level: | |||||
msg = color(msg) | |||||
break | |||||
return msg | |||||
class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): | |||||
def _open(self): | |||||
ensure_dir(os.path.dirname(self.baseFilename)) | |||||
return logging.handlers.RotatingFileHandler._open(self) | |||||
class MaxLevelFilter(logging.Filter): | |||||
def __init__(self, level): | |||||
self.level = level | |||||
def filter(self, record): | |||||
return record.levelno < self.level |
from __future__ import absolute_import | |||||
import contextlib | |||||
import errno | |||||
import io | |||||
import locale | |||||
# we have a submodule named 'logging' which would shadow this if we used the | |||||
# regular name: | |||||
import logging as std_logging | |||||
import os | |||||
import posixpath | |||||
import re | |||||
import shutil | |||||
import stat | |||||
import subprocess | |||||
import sys | |||||
import tarfile | |||||
import zipfile | |||||
from collections import deque | |||||
from pip._vendor import pkg_resources | |||||
# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is | |||||
# why we ignore the type on this import. | |||||
from pip._vendor.retrying import retry # type: ignore | |||||
from pip._vendor.six import PY2 | |||||
from pip._vendor.six.moves import input | |||||
from pip._internal.compat import console_to_str, expanduser, stdlib_pkgs | |||||
from pip._internal.exceptions import InstallationError | |||||
from pip._internal.locations import ( | |||||
running_under_virtualenv, site_packages, user_site, virtualenv_no_global, | |||||
write_delete_marker_file, | |||||
) | |||||
if PY2: | |||||
from io import BytesIO as StringIO | |||||
else: | |||||
from io import StringIO | |||||
__all__ = ['rmtree', 'display_path', 'backup_dir', | |||||
'ask', 'splitext', | |||||
'format_size', 'is_installable_dir', | |||||
'is_svn_page', 'file_contents', | |||||
'split_leading_dir', 'has_leading_dir', | |||||
'normalize_path', | |||||
'renames', 'get_prog', | |||||
'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess', | |||||
'captured_stdout', 'ensure_dir', | |||||
'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', | |||||
'get_installed_version'] | |||||
logger = std_logging.getLogger(__name__) | |||||
BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') | |||||
XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma') | |||||
ZIP_EXTENSIONS = ('.zip', '.whl') | |||||
TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') | |||||
ARCHIVE_EXTENSIONS = ( | |||||
ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS) | |||||
SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS | |||||
try: | |||||
import bz2 # noqa | |||||
SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS | |||||
except ImportError: | |||||
logger.debug('bz2 module is not available') | |||||
try: | |||||
# Only for Python 3.3+ | |||||
import lzma # noqa | |||||
SUPPORTED_EXTENSIONS += XZ_EXTENSIONS | |||||
except ImportError: | |||||
logger.debug('lzma module is not available') | |||||
def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs): | |||||
try: | |||||
return __import__(pkg_or_module_string) | |||||
except ImportError: | |||||
raise ExceptionType(*args, **kwargs) | |||||
def ensure_dir(path): | |||||
"""os.path.makedirs without EEXIST.""" | |||||
try: | |||||
os.makedirs(path) | |||||
except OSError as e: | |||||
if e.errno != errno.EEXIST: | |||||
raise | |||||
def get_prog(): | |||||
try: | |||||
prog = os.path.basename(sys.argv[0]) | |||||
if prog in ('__main__.py', '-c'): | |||||
return "%s -m pip" % sys.executable | |||||
else: | |||||
return prog | |||||
except (AttributeError, TypeError, IndexError): | |||||
pass | |||||
return 'pip' | |||||
# Retry every half second for up to 3 seconds | |||||
@retry(stop_max_delay=3000, wait_fixed=500) | |||||
def rmtree(dir, ignore_errors=False): | |||||
shutil.rmtree(dir, ignore_errors=ignore_errors, | |||||
onerror=rmtree_errorhandler) | |||||
def rmtree_errorhandler(func, path, exc_info): | |||||
"""On Windows, the files in .svn are read-only, so when rmtree() tries to | |||||
remove them, an exception is thrown. We catch that here, remove the | |||||
read-only attribute, and hopefully continue without problems.""" | |||||
# if file type currently read only | |||||
if os.stat(path).st_mode & stat.S_IREAD: | |||||
# convert to read/write | |||||
os.chmod(path, stat.S_IWRITE) | |||||
# use the original function to repeat the operation | |||||
func(path) | |||||
return | |||||
else: | |||||
raise | |||||
def display_path(path): | |||||
"""Gives the display value for a given path, making it relative to cwd | |||||
if possible.""" | |||||
path = os.path.normcase(os.path.abspath(path)) | |||||
if sys.version_info[0] == 2: | |||||
path = path.decode(sys.getfilesystemencoding(), 'replace') | |||||
path = path.encode(sys.getdefaultencoding(), 'replace') | |||||
if path.startswith(os.getcwd() + os.path.sep): | |||||
path = '.' + path[len(os.getcwd()):] | |||||
return path | |||||
def backup_dir(dir, ext='.bak'): | |||||
"""Figure out the name of a directory to back up the given dir to | |||||
(adding .bak, .bak2, etc)""" | |||||
n = 1 | |||||
extension = ext | |||||
while os.path.exists(dir + extension): | |||||
n += 1 | |||||
extension = ext + str(n) | |||||
return dir + extension | |||||
def ask_path_exists(message, options): | |||||
for action in os.environ.get('PIP_EXISTS_ACTION', '').split(): | |||||
if action in options: | |||||
return action | |||||
return ask(message, options) | |||||
def ask(message, options): | |||||
"""Ask the message interactively, with the given possible responses""" | |||||
while 1: | |||||
if os.environ.get('PIP_NO_INPUT'): | |||||
raise Exception( | |||||
'No input was expected ($PIP_NO_INPUT set); question: %s' % | |||||
message | |||||
) | |||||
response = input(message) | |||||
response = response.strip().lower() | |||||
if response not in options: | |||||
print( | |||||
'Your response (%r) was not one of the expected responses: ' | |||||
'%s' % (response, ', '.join(options)) | |||||
) | |||||
else: | |||||
return response | |||||
def format_size(bytes): | |||||
if bytes > 1000 * 1000: | |||||
return '%.1fMB' % (bytes / 1000.0 / 1000) | |||||
elif bytes > 10 * 1000: | |||||
return '%ikB' % (bytes / 1000) | |||||
elif bytes > 1000: | |||||
return '%.1fkB' % (bytes / 1000.0) | |||||
else: | |||||
return '%ibytes' % bytes | |||||
def is_installable_dir(path): | |||||
"""Return True if `path` is a directory containing a setup.py file.""" | |||||
if not os.path.isdir(path): | |||||
return False | |||||
setup_py = os.path.join(path, 'setup.py') | |||||
if os.path.isfile(setup_py): | |||||
return True | |||||
return False | |||||
def is_svn_page(html): | |||||
""" | |||||
Returns true if the page appears to be the index page of an svn repository | |||||
""" | |||||
return (re.search(r'<title>[^<]*Revision \d+:', html) and | |||||
re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) | |||||
def file_contents(filename): | |||||
with open(filename, 'rb') as fp: | |||||
return fp.read().decode('utf-8') | |||||
def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): | |||||
"""Yield pieces of data from a file-like object until EOF.""" | |||||
while True: | |||||
chunk = file.read(size) | |||||
if not chunk: | |||||
break | |||||
yield chunk | |||||
def split_leading_dir(path): | |||||
path = path.lstrip('/').lstrip('\\') | |||||
if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or | |||||
'\\' not in path): | |||||
return path.split('/', 1) | |||||
elif '\\' in path: | |||||
return path.split('\\', 1) | |||||
else: | |||||
return path, '' | |||||
def has_leading_dir(paths): | |||||
"""Returns true if all the paths have the same leading path name | |||||
(i.e., everything is in one subdirectory in an archive)""" | |||||
common_prefix = None | |||||
for path in paths: | |||||
prefix, rest = split_leading_dir(path) | |||||
if not prefix: | |||||
return False | |||||
elif common_prefix is None: | |||||
common_prefix = prefix | |||||
elif prefix != common_prefix: | |||||
return False | |||||
return True | |||||
def normalize_path(path, resolve_symlinks=True): | |||||
""" | |||||
Convert a path to its canonical, case-normalized, absolute version. | |||||
""" | |||||
path = expanduser(path) | |||||
if resolve_symlinks: | |||||
path = os.path.realpath(path) | |||||
else: | |||||
path = os.path.abspath(path) | |||||
return os.path.normcase(path) | |||||
def splitext(path): | |||||
"""Like os.path.splitext, but take off .tar too""" | |||||
base, ext = posixpath.splitext(path) | |||||
if base.lower().endswith('.tar'): | |||||
ext = base[-4:] + ext | |||||
base = base[:-4] | |||||
return base, ext | |||||
def renames(old, new): | |||||
"""Like os.renames(), but handles renaming across devices.""" | |||||
# Implementation borrowed from os.renames(). | |||||
head, tail = os.path.split(new) | |||||
if head and tail and not os.path.exists(head): | |||||
os.makedirs(head) | |||||
shutil.move(old, new) | |||||
head, tail = os.path.split(old) | |||||
if head and tail: | |||||
try: | |||||
os.removedirs(head) | |||||
except OSError: | |||||
pass | |||||
def is_local(path): | |||||
""" | |||||
Return True if path is within sys.prefix, if we're running in a virtualenv. | |||||
If we're not in a virtualenv, all paths are considered "local." | |||||
""" | |||||
if not running_under_virtualenv(): | |||||
return True | |||||
return normalize_path(path).startswith(normalize_path(sys.prefix)) | |||||
def dist_is_local(dist): | |||||
""" | |||||
Return True if given Distribution object is installed locally | |||||
(i.e. within current virtualenv). | |||||
Always True if we're not in a virtualenv. | |||||
""" | |||||
return is_local(dist_location(dist)) | |||||
def dist_in_usersite(dist): | |||||
""" | |||||
Return True if given Distribution is installed in user site. | |||||
""" | |||||
norm_path = normalize_path(dist_location(dist)) | |||||
return norm_path.startswith(normalize_path(user_site)) | |||||
def dist_in_site_packages(dist): | |||||
""" | |||||
Return True if given Distribution is installed in | |||||
sysconfig.get_python_lib(). | |||||
""" | |||||
return normalize_path( | |||||
dist_location(dist) | |||||
).startswith(normalize_path(site_packages)) | |||||
def dist_is_editable(dist): | |||||
"""Is distribution an editable install?""" | |||||
for path_item in sys.path: | |||||
egg_link = os.path.join(path_item, dist.project_name + '.egg-link') | |||||
if os.path.isfile(egg_link): | |||||
return True | |||||
return False | |||||
def get_installed_distributions(local_only=True, | |||||
skip=stdlib_pkgs, | |||||
include_editables=True, | |||||
editables_only=False, | |||||
user_only=False): | |||||
""" | |||||
Return a list of installed Distribution objects. | |||||
If ``local_only`` is True (default), only return installations | |||||
local to the current virtualenv, if in a virtualenv. | |||||
``skip`` argument is an iterable of lower-case project names to | |||||
ignore; defaults to stdlib_pkgs | |||||
If ``include_editables`` is False, don't report editables. | |||||
If ``editables_only`` is True , only report editables. | |||||
If ``user_only`` is True , only report installations in the user | |||||
site directory. | |||||
""" | |||||
if local_only: | |||||
local_test = dist_is_local | |||||
else: | |||||
def local_test(d): | |||||
return True | |||||
if include_editables: | |||||
def editable_test(d): | |||||
return True | |||||
else: | |||||
def editable_test(d): | |||||
return not dist_is_editable(d) | |||||
if editables_only: | |||||
def editables_only_test(d): | |||||
return dist_is_editable(d) | |||||
else: | |||||
def editables_only_test(d): | |||||
return True | |||||
if user_only: | |||||
user_test = dist_in_usersite | |||||
else: | |||||
def user_test(d): | |||||
return True | |||||
return [d for d in pkg_resources.working_set | |||||
if local_test(d) and | |||||
d.key not in skip and | |||||
editable_test(d) and | |||||
editables_only_test(d) and | |||||
user_test(d) | |||||
] | |||||
def egg_link_path(dist): | |||||
""" | |||||
Return the path for the .egg-link file if it exists, otherwise, None. | |||||
There's 3 scenarios: | |||||
1) not in a virtualenv | |||||
try to find in site.USER_SITE, then site_packages | |||||
2) in a no-global virtualenv | |||||
try to find in site_packages | |||||
3) in a yes-global virtualenv | |||||
try to find in site_packages, then site.USER_SITE | |||||
(don't look in global location) | |||||
For #1 and #3, there could be odd cases, where there's an egg-link in 2 | |||||
locations. | |||||
This method will just return the first one found. | |||||
""" | |||||
sites = [] | |||||
if running_under_virtualenv(): | |||||
if virtualenv_no_global(): | |||||
sites.append(site_packages) | |||||
else: | |||||
sites.append(site_packages) | |||||
if user_site: | |||||
sites.append(user_site) | |||||
else: | |||||
if user_site: | |||||
sites.append(user_site) | |||||
sites.append(site_packages) | |||||
for site in sites: | |||||
egglink = os.path.join(site, dist.project_name) + '.egg-link' | |||||
if os.path.isfile(egglink): | |||||
return egglink | |||||
def dist_location(dist): | |||||
""" | |||||
Get the site-packages location of this distribution. Generally | |||||
this is dist.location, except in the case of develop-installed | |||||
packages, where dist.location is the source code location, and we | |||||
want to know where the egg-link file is. | |||||
""" | |||||
egg_link = egg_link_path(dist) | |||||
if egg_link: | |||||
return egg_link | |||||
return dist.location | |||||
def current_umask(): | |||||
"""Get the current umask which involves having to set it temporarily.""" | |||||
mask = os.umask(0) | |||||
os.umask(mask) | |||||
return mask | |||||
def unzip_file(filename, location, flatten=True): | |||||
""" | |||||
Unzip the file (with path `filename`) to the destination `location`. All | |||||
files are written based on system defaults and umask (i.e. permissions are | |||||
not preserved), except that regular file members with any execute | |||||
permissions (user, group, or world) have "chmod +x" applied after being | |||||
written. Note that for windows, any execute changes using os.chmod are | |||||
no-ops per the python docs. | |||||
""" | |||||
ensure_dir(location) | |||||
zipfp = open(filename, 'rb') | |||||
try: | |||||
zip = zipfile.ZipFile(zipfp, allowZip64=True) | |||||
leading = has_leading_dir(zip.namelist()) and flatten | |||||
for info in zip.infolist(): | |||||
name = info.filename | |||||
data = zip.read(name) | |||||
fn = name | |||||
if leading: | |||||
fn = split_leading_dir(name)[1] | |||||
fn = os.path.join(location, fn) | |||||
dir = os.path.dirname(fn) | |||||
if fn.endswith('/') or fn.endswith('\\'): | |||||
# A directory | |||||
ensure_dir(fn) | |||||
else: | |||||
ensure_dir(dir) | |||||
fp = open(fn, 'wb') | |||||
try: | |||||
fp.write(data) | |||||
finally: | |||||
fp.close() | |||||
mode = info.external_attr >> 16 | |||||
# if mode and regular file and any execute permissions for | |||||
# user/group/world? | |||||
if mode and stat.S_ISREG(mode) and mode & 0o111: | |||||
# make dest file have execute for user/group/world | |||||
# (chmod +x) no-op on windows per python docs | |||||
os.chmod(fn, (0o777 - current_umask() | 0o111)) | |||||
finally: | |||||
zipfp.close() | |||||
def untar_file(filename, location): | |||||
""" | |||||
Untar the file (with path `filename`) to the destination `location`. | |||||
All files are written based on system defaults and umask (i.e. permissions | |||||
are not preserved), except that regular file members with any execute | |||||
permissions (user, group, or world) have "chmod +x" applied after being | |||||
written. Note that for windows, any execute changes using os.chmod are | |||||
no-ops per the python docs. | |||||
""" | |||||
ensure_dir(location) | |||||
if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): | |||||
mode = 'r:gz' | |||||
elif filename.lower().endswith(BZ2_EXTENSIONS): | |||||
mode = 'r:bz2' | |||||
elif filename.lower().endswith(XZ_EXTENSIONS): | |||||
mode = 'r:xz' | |||||
elif filename.lower().endswith('.tar'): | |||||
mode = 'r' | |||||
else: | |||||
logger.warning( | |||||
'Cannot determine compression type for file %s', filename, | |||||
) | |||||
mode = 'r:*' | |||||
tar = tarfile.open(filename, mode) | |||||
try: | |||||
# note: python<=2.5 doesn't seem to know about pax headers, filter them | |||||
leading = has_leading_dir([ | |||||
member.name for member in tar.getmembers() | |||||
if member.name != 'pax_global_header' | |||||
]) | |||||
for member in tar.getmembers(): | |||||
fn = member.name | |||||
if fn == 'pax_global_header': | |||||
continue | |||||
if leading: | |||||
fn = split_leading_dir(fn)[1] | |||||
path = os.path.join(location, fn) | |||||
if member.isdir(): | |||||
ensure_dir(path) | |||||
elif member.issym(): | |||||
try: | |||||
tar._extract_member(member, path) | |||||
except Exception as exc: | |||||
# Some corrupt tar files seem to produce this | |||||
# (specifically bad symlinks) | |||||
logger.warning( | |||||
'In the tar file %s the member %s is invalid: %s', | |||||
filename, member.name, exc, | |||||
) | |||||
continue | |||||
else: | |||||
try: | |||||
fp = tar.extractfile(member) | |||||
except (KeyError, AttributeError) as exc: | |||||
# Some corrupt tar files seem to produce this | |||||
# (specifically bad symlinks) | |||||
logger.warning( | |||||
'In the tar file %s the member %s is invalid: %s', | |||||
filename, member.name, exc, | |||||
) | |||||
continue | |||||
ensure_dir(os.path.dirname(path)) | |||||
with open(path, 'wb') as destfp: | |||||
shutil.copyfileobj(fp, destfp) | |||||
fp.close() | |||||
# Update the timestamp (useful for cython compiled files) | |||||
tar.utime(member, path) | |||||
# member have any execute permissions for user/group/world? | |||||
if member.mode & 0o111: | |||||
# make dest file have execute for user/group/world | |||||
# no-op on windows per python docs | |||||
os.chmod(path, (0o777 - current_umask() | 0o111)) | |||||
finally: | |||||
tar.close() | |||||
def unpack_file(filename, location, content_type, link): | |||||
filename = os.path.realpath(filename) | |||||
if (content_type == 'application/zip' or | |||||
filename.lower().endswith(ZIP_EXTENSIONS) or | |||||
zipfile.is_zipfile(filename)): | |||||
unzip_file( | |||||
filename, | |||||
location, | |||||
flatten=not filename.endswith('.whl') | |||||
) | |||||
elif (content_type == 'application/x-gzip' or | |||||
tarfile.is_tarfile(filename) or | |||||
filename.lower().endswith( | |||||
TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)): | |||||
untar_file(filename, location) | |||||
elif (content_type and content_type.startswith('text/html') and | |||||
is_svn_page(file_contents(filename))): | |||||
# We don't really care about this | |||||
from pip._internal.vcs.subversion import Subversion | |||||
Subversion('svn+' + link.url).unpack(location) | |||||
else: | |||||
# FIXME: handle? | |||||
# FIXME: magic signatures? | |||||
logger.critical( | |||||
'Cannot unpack file %s (downloaded from %s, content-type: %s); ' | |||||
'cannot detect archive format', | |||||
filename, location, content_type, | |||||
) | |||||
raise InstallationError( | |||||
'Cannot determine archive format of %s' % location | |||||
) | |||||
def call_subprocess(cmd, show_stdout=True, cwd=None, | |||||
on_returncode='raise', | |||||
command_desc=None, | |||||
extra_environ=None, unset_environ=None, spinner=None): | |||||
""" | |||||
Args: | |||||
unset_environ: an iterable of environment variable names to unset | |||||
prior to calling subprocess.Popen(). | |||||
""" | |||||
if unset_environ is None: | |||||
unset_environ = [] | |||||
# This function's handling of subprocess output is confusing and I | |||||
# previously broke it terribly, so as penance I will write a long comment | |||||
# explaining things. | |||||
# | |||||
# The obvious thing that affects output is the show_stdout= | |||||
# kwarg. show_stdout=True means, let the subprocess write directly to our | |||||
# stdout. Even though it is nominally the default, it is almost never used | |||||
# inside pip (and should not be used in new code without a very good | |||||
# reason); as of 2016-02-22 it is only used in a few places inside the VCS | |||||
# wrapper code. Ideally we should get rid of it entirely, because it | |||||
# creates a lot of complexity here for a rarely used feature. | |||||
# | |||||
# Most places in pip set show_stdout=False. What this means is: | |||||
# - We connect the child stdout to a pipe, which we read. | |||||
# - By default, we hide the output but show a spinner -- unless the | |||||
# subprocess exits with an error, in which case we show the output. | |||||
# - If the --verbose option was passed (= loglevel is DEBUG), then we show | |||||
# the output unconditionally. (But in this case we don't want to show | |||||
# the output a second time if it turns out that there was an error.) | |||||
# | |||||
# stderr is always merged with stdout (even if show_stdout=True). | |||||
if show_stdout: | |||||
stdout = None | |||||
else: | |||||
stdout = subprocess.PIPE | |||||
if command_desc is None: | |||||
cmd_parts = [] | |||||
for part in cmd: | |||||
if ' ' in part or '\n' in part or '"' in part or "'" in part: | |||||
part = '"%s"' % part.replace('"', '\\"') | |||||
cmd_parts.append(part) | |||||
command_desc = ' '.join(cmd_parts) | |||||
logger.debug("Running command %s", command_desc) | |||||
env = os.environ.copy() | |||||
if extra_environ: | |||||
env.update(extra_environ) | |||||
for name in unset_environ: | |||||
env.pop(name, None) | |||||
try: | |||||
proc = subprocess.Popen( | |||||
cmd, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, | |||||
stdout=stdout, cwd=cwd, env=env, | |||||
) | |||||
proc.stdin.close() | |||||
except Exception as exc: | |||||
logger.critical( | |||||
"Error %s while executing command %s", exc, command_desc, | |||||
) | |||||
raise | |||||
all_output = [] | |||||
if stdout is not None: | |||||
while True: | |||||
line = console_to_str(proc.stdout.readline()) | |||||
if not line: | |||||
break | |||||
line = line.rstrip() | |||||
all_output.append(line + '\n') | |||||
if logger.getEffectiveLevel() <= std_logging.DEBUG: | |||||
# Show the line immediately | |||||
logger.debug(line) | |||||
else: | |||||
# Update the spinner | |||||
if spinner is not None: | |||||
spinner.spin() | |||||
try: | |||||
proc.wait() | |||||
finally: | |||||
if proc.stdout: | |||||
proc.stdout.close() | |||||
if spinner is not None: | |||||
if proc.returncode: | |||||
spinner.finish("error") | |||||
else: | |||||
spinner.finish("done") | |||||
if proc.returncode: | |||||
if on_returncode == 'raise': | |||||
if (logger.getEffectiveLevel() > std_logging.DEBUG and | |||||
not show_stdout): | |||||
logger.info( | |||||
'Complete output from command %s:', command_desc, | |||||
) | |||||
logger.info( | |||||
''.join(all_output) + | |||||
'\n----------------------------------------' | |||||
) | |||||
raise InstallationError( | |||||
'Command "%s" failed with error code %s in %s' | |||||
% (command_desc, proc.returncode, cwd)) | |||||
elif on_returncode == 'warn': | |||||
logger.warning( | |||||
'Command "%s" had error code %s in %s', | |||||
command_desc, proc.returncode, cwd, | |||||
) | |||||
elif on_returncode == 'ignore': | |||||
pass | |||||
else: | |||||
raise ValueError('Invalid value: on_returncode=%s' % | |||||
repr(on_returncode)) | |||||
if not show_stdout: | |||||
return ''.join(all_output) | |||||
def read_text_file(filename): | |||||
"""Return the contents of *filename*. | |||||
Try to decode the file contents with utf-8, the preferred system encoding | |||||
(e.g., cp1252 on some Windows machines), and latin1, in that order. | |||||
Decoding a byte string with latin1 will never raise an error. In the worst | |||||
case, the returned string will contain some garbage characters. | |||||
""" | |||||
with open(filename, 'rb') as fp: | |||||
data = fp.read() | |||||
encodings = ['utf-8', locale.getpreferredencoding(False), 'latin1'] | |||||
for enc in encodings: | |||||
try: | |||||
data = data.decode(enc) | |||||
except UnicodeDecodeError: | |||||
continue | |||||
break | |||||
assert type(data) != bytes # Latin1 should have worked. | |||||
return data | |||||
def _make_build_dir(build_dir): | |||||
os.makedirs(build_dir) | |||||
write_delete_marker_file(build_dir) | |||||
class FakeFile(object): | |||||
"""Wrap a list of lines in an object with readline() to make | |||||
ConfigParser happy.""" | |||||
def __init__(self, lines): | |||||
self._gen = (l for l in lines) | |||||
def readline(self): | |||||
try: | |||||
try: | |||||
return next(self._gen) | |||||
except NameError: | |||||
return self._gen.next() | |||||
except StopIteration: | |||||
return '' | |||||
def __iter__(self): | |||||
return self._gen | |||||
class StreamWrapper(StringIO): | |||||
@classmethod | |||||
def from_stream(cls, orig_stream): | |||||
cls.orig_stream = orig_stream | |||||
return cls() | |||||
# compileall.compile_dir() needs stdout.encoding to print to stdout | |||||
@property | |||||
def encoding(self): | |||||
return self.orig_stream.encoding | |||||
@contextlib.contextmanager | |||||
def captured_output(stream_name): | |||||
"""Return a context manager used by captured_stdout/stdin/stderr | |||||
that temporarily replaces the sys stream *stream_name* with a StringIO. | |||||
Taken from Lib/support/__init__.py in the CPython repo. | |||||
""" | |||||
orig_stdout = getattr(sys, stream_name) | |||||
setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout)) | |||||
try: | |||||
yield getattr(sys, stream_name) | |||||
finally: | |||||
setattr(sys, stream_name, orig_stdout) | |||||
def captured_stdout(): | |||||
"""Capture the output of sys.stdout: | |||||
with captured_stdout() as stdout: | |||||
print('hello') | |||||
self.assertEqual(stdout.getvalue(), 'hello\n') | |||||
Taken from Lib/support/__init__.py in the CPython repo. | |||||
""" | |||||
return captured_output('stdout') | |||||
class cached_property(object): | |||||
"""A property that is only computed once per instance and then replaces | |||||
itself with an ordinary attribute. Deleting the attribute resets the | |||||
property. | |||||
Source: https://github.com/bottlepy/bottle/blob/0.11.5/bottle.py#L175 | |||||
""" | |||||
def __init__(self, func): | |||||
self.__doc__ = getattr(func, '__doc__') | |||||
self.func = func | |||||
def __get__(self, obj, cls): | |||||
if obj is None: | |||||
# We're being accessed from the class itself, not from an object | |||||
return self | |||||
value = obj.__dict__[self.func.__name__] = self.func(obj) | |||||
return value | |||||
def get_installed_version(dist_name, lookup_dirs=None): | |||||
"""Get the installed version of dist_name avoiding pkg_resources cache""" | |||||
# Create a requirement that we'll look for inside of setuptools. | |||||
req = pkg_resources.Requirement.parse(dist_name) | |||||
# We want to avoid having this cached, so we need to construct a new | |||||
# working set each time. | |||||
if lookup_dirs is None: | |||||
working_set = pkg_resources.WorkingSet() | |||||
else: | |||||
working_set = pkg_resources.WorkingSet(lookup_dirs) | |||||
# Get the installed distribution from our working set | |||||
dist = working_set.find(req) | |||||
# Check to see if we got an installed distribution or not, if we did | |||||
# we want to return it's version. | |||||
return dist.version if dist else None | |||||
def consume(iterator): | |||||
"""Consume an iterable at C speed.""" | |||||
deque(iterator, maxlen=0) | |||||
# Simulates an enum | |||||
def enum(*sequential, **named): | |||||
enums = dict(zip(sequential, range(len(sequential))), **named) | |||||
reverse = {value: key for key, value in enums.items()} | |||||
enums['reverse_mapping'] = reverse | |||||
return type('Enum', (), enums) |
from __future__ import absolute_import | |||||
import datetime | |||||
import json | |||||
import logging | |||||
import os.path | |||||
import sys | |||||
from pip._vendor import lockfile | |||||
from pip._vendor.packaging import version as packaging_version | |||||
from pip._internal.compat import WINDOWS | |||||
from pip._internal.index import PackageFinder | |||||
from pip._internal.locations import USER_CACHE_DIR, running_under_virtualenv | |||||
from pip._internal.utils.filesystem import check_path_owner | |||||
from pip._internal.utils.misc import ensure_dir, get_installed_version | |||||
SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" | |||||
logger = logging.getLogger(__name__) | |||||
class VirtualenvSelfCheckState(object): | |||||
def __init__(self): | |||||
self.statefile_path = os.path.join(sys.prefix, "pip-selfcheck.json") | |||||
# Load the existing state | |||||
try: | |||||
with open(self.statefile_path) as statefile: | |||||
self.state = json.load(statefile) | |||||
except (IOError, ValueError): | |||||
self.state = {} | |||||
def save(self, pypi_version, current_time): | |||||
# Attempt to write out our version check file | |||||
with open(self.statefile_path, "w") as statefile: | |||||
json.dump( | |||||
{ | |||||
"last_check": current_time.strftime(SELFCHECK_DATE_FMT), | |||||
"pypi_version": pypi_version, | |||||
}, | |||||
statefile, | |||||
sort_keys=True, | |||||
separators=(",", ":") | |||||
) | |||||
class GlobalSelfCheckState(object): | |||||
def __init__(self): | |||||
self.statefile_path = os.path.join(USER_CACHE_DIR, "selfcheck.json") | |||||
# Load the existing state | |||||
try: | |||||
with open(self.statefile_path) as statefile: | |||||
self.state = json.load(statefile)[sys.prefix] | |||||
except (IOError, ValueError, KeyError): | |||||
self.state = {} | |||||
def save(self, pypi_version, current_time): | |||||
# Check to make sure that we own the directory | |||||
if not check_path_owner(os.path.dirname(self.statefile_path)): | |||||
return | |||||
# Now that we've ensured the directory is owned by this user, we'll go | |||||
# ahead and make sure that all our directories are created. | |||||
ensure_dir(os.path.dirname(self.statefile_path)) | |||||
# Attempt to write out our version check file | |||||
with lockfile.LockFile(self.statefile_path): | |||||
if os.path.exists(self.statefile_path): | |||||
with open(self.statefile_path) as statefile: | |||||
state = json.load(statefile) | |||||
else: | |||||
state = {} | |||||
state[sys.prefix] = { | |||||
"last_check": current_time.strftime(SELFCHECK_DATE_FMT), | |||||
"pypi_version": pypi_version, | |||||
} | |||||
with open(self.statefile_path, "w") as statefile: | |||||
json.dump(state, statefile, sort_keys=True, | |||||
separators=(",", ":")) | |||||
def load_selfcheck_statefile(): | |||||
if running_under_virtualenv(): | |||||
return VirtualenvSelfCheckState() | |||||
else: | |||||
return GlobalSelfCheckState() | |||||
def pip_version_check(session, options): | |||||
"""Check for an update for pip. | |||||
Limit the frequency of checks to once per week. State is stored either in | |||||
the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix | |||||
of the pip script path. | |||||
""" | |||||
installed_version = get_installed_version("pip") | |||||
if not installed_version: | |||||
return | |||||
pip_version = packaging_version.parse(installed_version) | |||||
pypi_version = None | |||||
try: | |||||
state = load_selfcheck_statefile() | |||||
current_time = datetime.datetime.utcnow() | |||||
# Determine if we need to refresh the state | |||||
if "last_check" in state.state and "pypi_version" in state.state: | |||||
last_check = datetime.datetime.strptime( | |||||
state.state["last_check"], | |||||
SELFCHECK_DATE_FMT | |||||
) | |||||
if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60: | |||||
pypi_version = state.state["pypi_version"] | |||||
# Refresh the version if we need to or just see if we need to warn | |||||
if pypi_version is None: | |||||
# Lets use PackageFinder to see what the latest pip version is | |||||
finder = PackageFinder( | |||||
find_links=options.find_links, | |||||
index_urls=[options.index_url] + options.extra_index_urls, | |||||
allow_all_prereleases=False, # Explicitly set to False | |||||
trusted_hosts=options.trusted_hosts, | |||||
process_dependency_links=options.process_dependency_links, | |||||
session=session, | |||||
) | |||||
all_candidates = finder.find_all_candidates("pip") | |||||
if not all_candidates: | |||||
return | |||||
pypi_version = str( | |||||
max(all_candidates, key=lambda c: c.version).version | |||||
) | |||||
# save that we've performed a check | |||||
state.save(pypi_version, current_time) | |||||
remote_version = packaging_version.parse(pypi_version) | |||||
# Determine if our pypi_version is older | |||||
if (pip_version < remote_version and | |||||
pip_version.base_version != remote_version.base_version): | |||||
# Advise "python -m pip" on Windows to avoid issues | |||||
# with overwriting pip.exe. | |||||
if WINDOWS: | |||||
pip_cmd = "python -m pip" | |||||
else: | |||||
pip_cmd = "pip" | |||||
logger.warning( | |||||
"You are using pip version %s, however version %s is " | |||||
"available.\nYou should consider upgrading via the " | |||||
"'%s install --upgrade pip' command.", | |||||
pip_version, pypi_version, pip_cmd | |||||
) | |||||
except Exception: | |||||
logger.debug( | |||||
"There was an error checking the latest version of pip", | |||||
exc_info=True, | |||||
) |
from __future__ import absolute_import | |||||
import logging | |||||
import sys | |||||
from email.parser import FeedParser # type: ignore | |||||
from pip._vendor import pkg_resources | |||||
from pip._vendor.packaging import specifiers, version | |||||
from pip._internal import exceptions | |||||
logger = logging.getLogger(__name__) | |||||
def check_requires_python(requires_python): | |||||
""" | |||||
Check if the python version in use match the `requires_python` specifier. | |||||
Returns `True` if the version of python in use matches the requirement. | |||||
Returns `False` if the version of python in use does not matches the | |||||
requirement. | |||||
Raises an InvalidSpecifier if `requires_python` have an invalid format. | |||||
""" | |||||
if requires_python is None: | |||||
# The package provides no information | |||||
return True | |||||
requires_python_specifier = specifiers.SpecifierSet(requires_python) | |||||
# We only use major.minor.micro | |||||
python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) | |||||
return python_version in requires_python_specifier | |||||
def get_metadata(dist): | |||||
if (isinstance(dist, pkg_resources.DistInfoDistribution) and | |||||
dist.has_metadata('METADATA')): | |||||
return dist.get_metadata('METADATA') | |||||
elif dist.has_metadata('PKG-INFO'): | |||||
return dist.get_metadata('PKG-INFO') | |||||
def check_dist_requires_python(dist): | |||||
metadata = get_metadata(dist) | |||||
feed_parser = FeedParser() | |||||
feed_parser.feed(metadata) | |||||
pkg_info_dict = feed_parser.close() | |||||
requires_python = pkg_info_dict.get('Requires-Python') | |||||
try: | |||||
if not check_requires_python(requires_python): | |||||
raise exceptions.UnsupportedPythonVersion( | |||||
"%s requires Python '%s' but the running Python is %s" % ( | |||||
dist.project_name, | |||||
requires_python, | |||||
'.'.join(map(str, sys.version_info[:3])),) | |||||
) | |||||
except specifiers.InvalidSpecifier as e: | |||||
logger.warning( | |||||
"Package %s has an invalid Requires-Python entry %s - %s", | |||||
dist.project_name, requires_python, e, | |||||
) | |||||
return | |||||
def get_installer(dist): | |||||
if dist.has_metadata('INSTALLER'): | |||||
for line in dist.get_metadata_lines('INSTALLER'): | |||||
if line.strip(): | |||||
return line.strip() | |||||
return '' |
# Shim to wrap setup.py invocation with setuptools | |||||
SETUPTOOLS_SHIM = ( | |||||
"import setuptools, tokenize;__file__=%r;" | |||||
"f=getattr(tokenize, 'open', open)(__file__);" | |||||
"code=f.read().replace('\\r\\n', '\\n');" | |||||
"f.close();" | |||||
"exec(compile(code, __file__, 'exec'))" | |||||
) |
from __future__ import absolute_import | |||||
import logging | |||||
import os.path | |||||
import tempfile | |||||
from pip._internal.utils.misc import rmtree | |||||
logger = logging.getLogger(__name__) | |||||
class TempDirectory(object): | |||||
"""Helper class that owns and cleans up a temporary directory. | |||||
This class can be used as a context manager or as an OO representation of a | |||||
temporary directory. | |||||
Attributes: | |||||
path | |||||
Location to the created temporary directory or None | |||||
delete | |||||
Whether the directory should be deleted when exiting | |||||
(when used as a contextmanager) | |||||
Methods: | |||||
create() | |||||
Creates a temporary directory and stores its path in the path | |||||
attribute. | |||||
cleanup() | |||||
Deletes the temporary directory and sets path attribute to None | |||||
When used as a context manager, a temporary directory is created on | |||||
entering the context and, if the delete attribute is True, on exiting the | |||||
context the created directory is deleted. | |||||
""" | |||||
def __init__(self, path=None, delete=None, kind="temp"): | |||||
super(TempDirectory, self).__init__() | |||||
if path is None and delete is None: | |||||
# If we were not given an explicit directory, and we were not given | |||||
# an explicit delete option, then we'll default to deleting. | |||||
delete = True | |||||
self.path = path | |||||
self.delete = delete | |||||
self.kind = kind | |||||
def __repr__(self): | |||||
return "<{} {!r}>".format(self.__class__.__name__, self.path) | |||||
def __enter__(self): | |||||
self.create() | |||||
return self | |||||
def __exit__(self, exc, value, tb): | |||||
if self.delete: | |||||
self.cleanup() | |||||
def create(self): | |||||
"""Create a temporary directory and store it's path in self.path | |||||
""" | |||||
if self.path is not None: | |||||
logger.debug( | |||||
"Skipped creation of temporary directory: {}".format(self.path) | |||||
) | |||||
return | |||||
# We realpath here because some systems have their default tmpdir | |||||
# symlinked to another directory. This tends to confuse build | |||||
# scripts, so we canonicalize the path by traversing potential | |||||
# symlinks here. | |||||
self.path = os.path.realpath( | |||||
tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) | |||||
) | |||||
logger.debug("Created temporary directory: {}".format(self.path)) | |||||
def cleanup(self): | |||||
"""Remove the temporary directory created and reset state | |||||
""" | |||||
if self.path is not None and os.path.exists(self.path): | |||||
rmtree(self.path) | |||||
self.path = None |
"""For neatly implementing static typing in pip. | |||||
`mypy` - the static type analysis tool we use - uses the `typing` module, which | |||||
provides core functionality fundamental to mypy's functioning. | |||||
Generally, `typing` would be imported at runtime and used in that fashion - | |||||
it acts as a no-op at runtime and does not have any run-time overhead by | |||||
design. | |||||
As it turns out, `typing` is not vendorable - it uses separate sources for | |||||
Python 2/Python 3. Thus, this codebase can not expect it to be present. | |||||
To work around this, mypy allows the typing import to be behind a False-y | |||||
optional to prevent it from running at runtime and type-comments can be used | |||||
to remove the need for the types to be accessible directly during runtime. | |||||
This module provides the False-y guard in a nicely named fashion so that a | |||||
curious maintainer can reach here to read this. | |||||
In pip, all static-typing related imports should be guarded as follows: | |||||
from pip.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import ... | |||||
Ref: https://github.com/python/mypy/issues/3216 | |||||
""" | |||||
MYPY_CHECK_RUNNING = False |
from __future__ import absolute_import, division | |||||
import contextlib | |||||
import itertools | |||||
import logging | |||||
import sys | |||||
import time | |||||
from signal import SIGINT, default_int_handler, signal | |||||
from pip._vendor import six | |||||
from pip._vendor.progress.bar import ( | |||||
Bar, ChargingBar, FillingCirclesBar, FillingSquaresBar, IncrementalBar, | |||||
ShadyBar, | |||||
) | |||||
from pip._vendor.progress.helpers import HIDE_CURSOR, SHOW_CURSOR, WritelnMixin | |||||
from pip._vendor.progress.spinner import Spinner | |||||
from pip._internal.compat import WINDOWS | |||||
from pip._internal.utils.logging import get_indentation | |||||
from pip._internal.utils.misc import format_size | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Any | |||||
try: | |||||
from pip._vendor import colorama | |||||
# Lots of different errors can come from this, including SystemError and | |||||
# ImportError. | |||||
except Exception: | |||||
colorama = None | |||||
logger = logging.getLogger(__name__) | |||||
def _select_progress_class(preferred, fallback): | |||||
encoding = getattr(preferred.file, "encoding", None) | |||||
# If we don't know what encoding this file is in, then we'll just assume | |||||
# that it doesn't support unicode and use the ASCII bar. | |||||
if not encoding: | |||||
return fallback | |||||
# Collect all of the possible characters we want to use with the preferred | |||||
# bar. | |||||
characters = [ | |||||
getattr(preferred, "empty_fill", six.text_type()), | |||||
getattr(preferred, "fill", six.text_type()), | |||||
] | |||||
characters += list(getattr(preferred, "phases", [])) | |||||
# Try to decode the characters we're using for the bar using the encoding | |||||
# of the given file, if this works then we'll assume that we can use the | |||||
# fancier bar and if not we'll fall back to the plaintext bar. | |||||
try: | |||||
six.text_type().join(characters).encode(encoding) | |||||
except UnicodeEncodeError: | |||||
return fallback | |||||
else: | |||||
return preferred | |||||
_BaseBar = _select_progress_class(IncrementalBar, Bar) # type: Any | |||||
class InterruptibleMixin(object): | |||||
""" | |||||
Helper to ensure that self.finish() gets called on keyboard interrupt. | |||||
This allows downloads to be interrupted without leaving temporary state | |||||
(like hidden cursors) behind. | |||||
This class is similar to the progress library's existing SigIntMixin | |||||
helper, but as of version 1.2, that helper has the following problems: | |||||
1. It calls sys.exit(). | |||||
2. It discards the existing SIGINT handler completely. | |||||
3. It leaves its own handler in place even after an uninterrupted finish, | |||||
which will have unexpected delayed effects if the user triggers an | |||||
unrelated keyboard interrupt some time after a progress-displaying | |||||
download has already completed, for example. | |||||
""" | |||||
def __init__(self, *args, **kwargs): | |||||
""" | |||||
Save the original SIGINT handler for later. | |||||
""" | |||||
super(InterruptibleMixin, self).__init__(*args, **kwargs) | |||||
self.original_handler = signal(SIGINT, self.handle_sigint) | |||||
# If signal() returns None, the previous handler was not installed from | |||||
# Python, and we cannot restore it. This probably should not happen, | |||||
# but if it does, we must restore something sensible instead, at least. | |||||
# The least bad option should be Python's default SIGINT handler, which | |||||
# just raises KeyboardInterrupt. | |||||
if self.original_handler is None: | |||||
self.original_handler = default_int_handler | |||||
def finish(self): | |||||
""" | |||||
Restore the original SIGINT handler after finishing. | |||||
This should happen regardless of whether the progress display finishes | |||||
normally, or gets interrupted. | |||||
""" | |||||
super(InterruptibleMixin, self).finish() | |||||
signal(SIGINT, self.original_handler) | |||||
def handle_sigint(self, signum, frame): | |||||
""" | |||||
Call self.finish() before delegating to the original SIGINT handler. | |||||
This handler should only be in place while the progress display is | |||||
active. | |||||
""" | |||||
self.finish() | |||||
self.original_handler(signum, frame) | |||||
class SilentBar(Bar): | |||||
def update(self): | |||||
pass | |||||
class BlueEmojiBar(IncrementalBar): | |||||
suffix = "%(percent)d%%" | |||||
bar_prefix = " " | |||||
bar_suffix = " " | |||||
phases = (u"\U0001F539", u"\U0001F537", u"\U0001F535") # type: Any | |||||
class DownloadProgressMixin(object): | |||||
def __init__(self, *args, **kwargs): | |||||
super(DownloadProgressMixin, self).__init__(*args, **kwargs) | |||||
self.message = (" " * (get_indentation() + 2)) + self.message | |||||
@property | |||||
def downloaded(self): | |||||
return format_size(self.index) | |||||
@property | |||||
def download_speed(self): | |||||
# Avoid zero division errors... | |||||
if self.avg == 0.0: | |||||
return "..." | |||||
return format_size(1 / self.avg) + "/s" | |||||
@property | |||||
def pretty_eta(self): | |||||
if self.eta: | |||||
return "eta %s" % self.eta_td | |||||
return "" | |||||
def iter(self, it, n=1): | |||||
for x in it: | |||||
yield x | |||||
self.next(n) | |||||
self.finish() | |||||
class WindowsMixin(object): | |||||
def __init__(self, *args, **kwargs): | |||||
# The Windows terminal does not support the hide/show cursor ANSI codes | |||||
# even with colorama. So we'll ensure that hide_cursor is False on | |||||
# Windows. | |||||
# This call neds to go before the super() call, so that hide_cursor | |||||
# is set in time. The base progress bar class writes the "hide cursor" | |||||
# code to the terminal in its init, so if we don't set this soon | |||||
# enough, we get a "hide" with no corresponding "show"... | |||||
if WINDOWS and self.hide_cursor: | |||||
self.hide_cursor = False | |||||
super(WindowsMixin, self).__init__(*args, **kwargs) | |||||
# Check if we are running on Windows and we have the colorama module, | |||||
# if we do then wrap our file with it. | |||||
if WINDOWS and colorama: | |||||
self.file = colorama.AnsiToWin32(self.file) | |||||
# The progress code expects to be able to call self.file.isatty() | |||||
# but the colorama.AnsiToWin32() object doesn't have that, so we'll | |||||
# add it. | |||||
self.file.isatty = lambda: self.file.wrapped.isatty() | |||||
# The progress code expects to be able to call self.file.flush() | |||||
# but the colorama.AnsiToWin32() object doesn't have that, so we'll | |||||
# add it. | |||||
self.file.flush = lambda: self.file.wrapped.flush() | |||||
class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, | |||||
DownloadProgressMixin): | |||||
file = sys.stdout | |||||
message = "%(percent)d%%" | |||||
suffix = "%(downloaded)s %(download_speed)s %(pretty_eta)s" | |||||
# NOTE: The "type: ignore" comments on the following classes are there to | |||||
# work around https://github.com/python/typing/issues/241 | |||||
class DefaultDownloadProgressBar(BaseDownloadProgressBar, | |||||
_BaseBar): # type: ignore | |||||
pass | |||||
class DownloadSilentBar(BaseDownloadProgressBar, SilentBar): # type: ignore | |||||
pass | |||||
class DownloadIncrementalBar(BaseDownloadProgressBar, # type: ignore | |||||
IncrementalBar): | |||||
pass | |||||
class DownloadChargingBar(BaseDownloadProgressBar, # type: ignore | |||||
ChargingBar): | |||||
pass | |||||
class DownloadShadyBar(BaseDownloadProgressBar, ShadyBar): # type: ignore | |||||
pass | |||||
class DownloadFillingSquaresBar(BaseDownloadProgressBar, # type: ignore | |||||
FillingSquaresBar): | |||||
pass | |||||
class DownloadFillingCirclesBar(BaseDownloadProgressBar, # type: ignore | |||||
FillingCirclesBar): | |||||
pass | |||||
class DownloadBlueEmojiProgressBar(BaseDownloadProgressBar, # type: ignore | |||||
BlueEmojiBar): | |||||
pass | |||||
class DownloadProgressSpinner(WindowsMixin, InterruptibleMixin, | |||||
DownloadProgressMixin, WritelnMixin, Spinner): | |||||
file = sys.stdout | |||||
suffix = "%(downloaded)s %(download_speed)s" | |||||
def next_phase(self): | |||||
if not hasattr(self, "_phaser"): | |||||
self._phaser = itertools.cycle(self.phases) | |||||
return next(self._phaser) | |||||
def update(self): | |||||
message = self.message % self | |||||
phase = self.next_phase() | |||||
suffix = self.suffix % self | |||||
line = ''.join([ | |||||
message, | |||||
" " if message else "", | |||||
phase, | |||||
" " if suffix else "", | |||||
suffix, | |||||
]) | |||||
self.writeln(line) | |||||
BAR_TYPES = { | |||||
"off": (DownloadSilentBar, DownloadSilentBar), | |||||
"on": (DefaultDownloadProgressBar, DownloadProgressSpinner), | |||||
"ascii": (DownloadIncrementalBar, DownloadProgressSpinner), | |||||
"pretty": (DownloadFillingCirclesBar, DownloadProgressSpinner), | |||||
"emoji": (DownloadBlueEmojiProgressBar, DownloadProgressSpinner) | |||||
} | |||||
def DownloadProgressProvider(progress_bar, max=None): | |||||
if max is None or max == 0: | |||||
return BAR_TYPES[progress_bar][1]().iter | |||||
else: | |||||
return BAR_TYPES[progress_bar][0](max=max).iter | |||||
################################################################ | |||||
# Generic "something is happening" spinners | |||||
# | |||||
# We don't even try using progress.spinner.Spinner here because it's actually | |||||
# simpler to reimplement from scratch than to coerce their code into doing | |||||
# what we need. | |||||
################################################################ | |||||
@contextlib.contextmanager | |||||
def hidden_cursor(file): | |||||
# The Windows terminal does not support the hide/show cursor ANSI codes, | |||||
# even via colorama. So don't even try. | |||||
if WINDOWS: | |||||
yield | |||||
# We don't want to clutter the output with control characters if we're | |||||
# writing to a file, or if the user is running with --quiet. | |||||
# See https://github.com/pypa/pip/issues/3418 | |||||
elif not file.isatty() or logger.getEffectiveLevel() > logging.INFO: | |||||
yield | |||||
else: | |||||
file.write(HIDE_CURSOR) | |||||
try: | |||||
yield | |||||
finally: | |||||
file.write(SHOW_CURSOR) | |||||
class RateLimiter(object): | |||||
def __init__(self, min_update_interval_seconds): | |||||
self._min_update_interval_seconds = min_update_interval_seconds | |||||
self._last_update = 0 | |||||
def ready(self): | |||||
now = time.time() | |||||
delta = now - self._last_update | |||||
return delta >= self._min_update_interval_seconds | |||||
def reset(self): | |||||
self._last_update = time.time() | |||||
class InteractiveSpinner(object): | |||||
def __init__(self, message, file=None, spin_chars="-\\|/", | |||||
# Empirically, 8 updates/second looks nice | |||||
min_update_interval_seconds=0.125): | |||||
self._message = message | |||||
if file is None: | |||||
file = sys.stdout | |||||
self._file = file | |||||
self._rate_limiter = RateLimiter(min_update_interval_seconds) | |||||
self._finished = False | |||||
self._spin_cycle = itertools.cycle(spin_chars) | |||||
self._file.write(" " * get_indentation() + self._message + " ... ") | |||||
self._width = 0 | |||||
def _write(self, status): | |||||
assert not self._finished | |||||
# Erase what we wrote before by backspacing to the beginning, writing | |||||
# spaces to overwrite the old text, and then backspacing again | |||||
backup = "\b" * self._width | |||||
self._file.write(backup + " " * self._width + backup) | |||||
# Now we have a blank slate to add our status | |||||
self._file.write(status) | |||||
self._width = len(status) | |||||
self._file.flush() | |||||
self._rate_limiter.reset() | |||||
def spin(self): | |||||
if self._finished: | |||||
return | |||||
if not self._rate_limiter.ready(): | |||||
return | |||||
self._write(next(self._spin_cycle)) | |||||
def finish(self, final_status): | |||||
if self._finished: | |||||
return | |||||
self._write(final_status) | |||||
self._file.write("\n") | |||||
self._file.flush() | |||||
self._finished = True | |||||
# Used for dumb terminals, non-interactive installs (no tty), etc. | |||||
# We still print updates occasionally (once every 60 seconds by default) to | |||||
# act as a keep-alive for systems like Travis-CI that take lack-of-output as | |||||
# an indication that a task has frozen. | |||||
class NonInteractiveSpinner(object): | |||||
def __init__(self, message, min_update_interval_seconds=60): | |||||
self._message = message | |||||
self._finished = False | |||||
self._rate_limiter = RateLimiter(min_update_interval_seconds) | |||||
self._update("started") | |||||
def _update(self, status): | |||||
assert not self._finished | |||||
self._rate_limiter.reset() | |||||
logger.info("%s: %s", self._message, status) | |||||
def spin(self): | |||||
if self._finished: | |||||
return | |||||
if not self._rate_limiter.ready(): | |||||
return | |||||
self._update("still running...") | |||||
def finish(self, final_status): | |||||
if self._finished: | |||||
return | |||||
self._update("finished with status '%s'" % (final_status,)) | |||||
self._finished = True | |||||
@contextlib.contextmanager | |||||
def open_spinner(message): | |||||
# Interactive spinner goes directly to sys.stdout rather than being routed | |||||
# through the logging system, but it acts like it has level INFO, | |||||
# i.e. it's only displayed if we're at level INFO or better. | |||||
# Non-interactive spinner goes through the logging system, so it is always | |||||
# in sync with logging configuration. | |||||
if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO: | |||||
spinner = InteractiveSpinner(message) | |||||
else: | |||||
spinner = NonInteractiveSpinner(message) | |||||
try: | |||||
with hidden_cursor(sys.stdout): | |||||
yield spinner | |||||
except KeyboardInterrupt: | |||||
spinner.finish("canceled") | |||||
raise | |||||
except Exception: | |||||
spinner.finish("error") | |||||
raise | |||||
else: | |||||
spinner.finish("done") |
"""Handles all VCS (version control) support""" | |||||
from __future__ import absolute_import | |||||
import copy | |||||
import errno | |||||
import logging | |||||
import os | |||||
import shutil | |||||
import sys | |||||
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
from pip._internal.exceptions import BadCommand | |||||
from pip._internal.utils.misc import ( | |||||
display_path, backup_dir, call_subprocess, rmtree, ask_path_exists, | |||||
) | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Dict, Optional, Tuple | |||||
from pip._internal.basecommand import Command | |||||
__all__ = ['vcs', 'get_src_requirement'] | |||||
logger = logging.getLogger(__name__) | |||||
class RevOptions(object): | |||||
""" | |||||
Encapsulates a VCS-specific revision to install, along with any VCS | |||||
install options. | |||||
Instances of this class should be treated as if immutable. | |||||
""" | |||||
def __init__(self, vcs, rev=None, extra_args=None): | |||||
""" | |||||
Args: | |||||
vcs: a VersionControl object. | |||||
rev: the name of the revision to install. | |||||
extra_args: a list of extra options. | |||||
""" | |||||
if extra_args is None: | |||||
extra_args = [] | |||||
self.extra_args = extra_args | |||||
self.rev = rev | |||||
self.vcs = vcs | |||||
def __repr__(self): | |||||
return '<RevOptions {}: rev={!r}>'.format(self.vcs.name, self.rev) | |||||
@property | |||||
def arg_rev(self): | |||||
if self.rev is None: | |||||
return self.vcs.default_arg_rev | |||||
return self.rev | |||||
def to_args(self): | |||||
""" | |||||
Return the VCS-specific command arguments. | |||||
""" | |||||
args = [] | |||||
rev = self.arg_rev | |||||
if rev is not None: | |||||
args += self.vcs.get_base_rev_args(rev) | |||||
args += self.extra_args | |||||
return args | |||||
def to_display(self): | |||||
if not self.rev: | |||||
return '' | |||||
return ' (to revision {})'.format(self.rev) | |||||
def make_new(self, rev): | |||||
""" | |||||
Make a copy of the current instance, but with a new rev. | |||||
Args: | |||||
rev: the name of the revision for the new object. | |||||
""" | |||||
return self.vcs.make_rev_options(rev, extra_args=self.extra_args) | |||||
class VcsSupport(object): | |||||
_registry = {} # type: Dict[str, Command] | |||||
schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] | |||||
def __init__(self): | |||||
# Register more schemes with urlparse for various version control | |||||
# systems | |||||
urllib_parse.uses_netloc.extend(self.schemes) | |||||
# Python >= 2.7.4, 3.3 doesn't have uses_fragment | |||||
if getattr(urllib_parse, 'uses_fragment', None): | |||||
urllib_parse.uses_fragment.extend(self.schemes) | |||||
super(VcsSupport, self).__init__() | |||||
def __iter__(self): | |||||
return self._registry.__iter__() | |||||
@property | |||||
def backends(self): | |||||
return list(self._registry.values()) | |||||
@property | |||||
def dirnames(self): | |||||
return [backend.dirname for backend in self.backends] | |||||
@property | |||||
def all_schemes(self): | |||||
schemes = [] | |||||
for backend in self.backends: | |||||
schemes.extend(backend.schemes) | |||||
return schemes | |||||
def register(self, cls): | |||||
if not hasattr(cls, 'name'): | |||||
logger.warning('Cannot register VCS %s', cls.__name__) | |||||
return | |||||
if cls.name not in self._registry: | |||||
self._registry[cls.name] = cls | |||||
logger.debug('Registered VCS backend: %s', cls.name) | |||||
def unregister(self, cls=None, name=None): | |||||
if name in self._registry: | |||||
del self._registry[name] | |||||
elif cls in self._registry.values(): | |||||
del self._registry[cls.name] | |||||
else: | |||||
logger.warning('Cannot unregister because no class or name given') | |||||
def get_backend_name(self, location): | |||||
""" | |||||
Return the name of the version control backend if found at given | |||||
location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') | |||||
""" | |||||
for vc_type in self._registry.values(): | |||||
if vc_type.controls_location(location): | |||||
logger.debug('Determine that %s uses VCS: %s', | |||||
location, vc_type.name) | |||||
return vc_type.name | |||||
return None | |||||
def get_backend(self, name): | |||||
name = name.lower() | |||||
if name in self._registry: | |||||
return self._registry[name] | |||||
def get_backend_from_location(self, location): | |||||
vc_type = self.get_backend_name(location) | |||||
if vc_type: | |||||
return self.get_backend(vc_type) | |||||
return None | |||||
vcs = VcsSupport() | |||||
class VersionControl(object): | |||||
name = '' | |||||
dirname = '' | |||||
# List of supported schemes for this Version Control | |||||
schemes = () # type: Tuple[str, ...] | |||||
# Iterable of environment variable names to pass to call_subprocess(). | |||||
unset_environ = () # type: Tuple[str, ...] | |||||
default_arg_rev = None # type: Optional[str] | |||||
def __init__(self, url=None, *args, **kwargs): | |||||
self.url = url | |||||
super(VersionControl, self).__init__(*args, **kwargs) | |||||
def get_base_rev_args(self, rev): | |||||
""" | |||||
Return the base revision arguments for a vcs command. | |||||
Args: | |||||
rev: the name of a revision to install. Cannot be None. | |||||
""" | |||||
raise NotImplementedError | |||||
def make_rev_options(self, rev=None, extra_args=None): | |||||
""" | |||||
Return a RevOptions object. | |||||
Args: | |||||
rev: the name of a revision to install. | |||||
extra_args: a list of extra options. | |||||
""" | |||||
return RevOptions(self, rev, extra_args=extra_args) | |||||
def _is_local_repository(self, repo): | |||||
""" | |||||
posix absolute paths start with os.path.sep, | |||||
win32 ones start with drive (like c:\\folder) | |||||
""" | |||||
drive, tail = os.path.splitdrive(repo) | |||||
return repo.startswith(os.path.sep) or drive | |||||
# See issue #1083 for why this method was introduced: | |||||
# https://github.com/pypa/pip/issues/1083 | |||||
def translate_egg_surname(self, surname): | |||||
# For example, Django has branches of the form "stable/1.7.x". | |||||
return surname.replace('/', '_') | |||||
def export(self, location): | |||||
""" | |||||
Export the repository at the url to the destination location | |||||
i.e. only download the files, without vcs informations | |||||
""" | |||||
raise NotImplementedError | |||||
def get_url_rev(self): | |||||
""" | |||||
Returns the correct repository URL and revision by parsing the given | |||||
repository URL | |||||
""" | |||||
error_message = ( | |||||
"Sorry, '%s' is a malformed VCS url. " | |||||
"The format is <vcs>+<protocol>://<url>, " | |||||
"e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" | |||||
) | |||||
assert '+' in self.url, error_message % self.url | |||||
url = self.url.split('+', 1)[1] | |||||
scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) | |||||
rev = None | |||||
if '@' in path: | |||||
path, rev = path.rsplit('@', 1) | |||||
url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) | |||||
return url, rev | |||||
def get_info(self, location): | |||||
""" | |||||
Returns (url, revision), where both are strings | |||||
""" | |||||
assert not location.rstrip('/').endswith(self.dirname), \ | |||||
'Bad directory: %s' % location | |||||
return self.get_url(location), self.get_revision(location) | |||||
def normalize_url(self, url): | |||||
""" | |||||
Normalize a URL for comparison by unquoting it and removing any | |||||
trailing slash. | |||||
""" | |||||
return urllib_parse.unquote(url).rstrip('/') | |||||
def compare_urls(self, url1, url2): | |||||
""" | |||||
Compare two repo URLs for identity, ignoring incidental differences. | |||||
""" | |||||
return (self.normalize_url(url1) == self.normalize_url(url2)) | |||||
def obtain(self, dest): | |||||
""" | |||||
Called when installing or updating an editable package, takes the | |||||
source path of the checkout. | |||||
""" | |||||
raise NotImplementedError | |||||
def switch(self, dest, url, rev_options): | |||||
""" | |||||
Switch the repo at ``dest`` to point to ``URL``. | |||||
Args: | |||||
rev_options: a RevOptions object. | |||||
""" | |||||
raise NotImplementedError | |||||
def update(self, dest, rev_options): | |||||
""" | |||||
Update an already-existing repo to the given ``rev_options``. | |||||
Args: | |||||
rev_options: a RevOptions object. | |||||
""" | |||||
raise NotImplementedError | |||||
def is_commit_id_equal(self, dest, name): | |||||
""" | |||||
Return whether the id of the current commit equals the given name. | |||||
Args: | |||||
dest: the repository directory. | |||||
name: a string name. | |||||
""" | |||||
raise NotImplementedError | |||||
def check_destination(self, dest, url, rev_options): | |||||
""" | |||||
Prepare a location to receive a checkout/clone. | |||||
Return True if the location is ready for (and requires) a | |||||
checkout/clone, False otherwise. | |||||
Args: | |||||
rev_options: a RevOptions object. | |||||
""" | |||||
checkout = True | |||||
prompt = False | |||||
rev_display = rev_options.to_display() | |||||
if os.path.exists(dest): | |||||
checkout = False | |||||
if os.path.exists(os.path.join(dest, self.dirname)): | |||||
existing_url = self.get_url(dest) | |||||
if self.compare_urls(existing_url, url): | |||||
logger.debug( | |||||
'%s in %s exists, and has correct URL (%s)', | |||||
self.repo_name.title(), | |||||
display_path(dest), | |||||
url, | |||||
) | |||||
if not self.is_commit_id_equal(dest, rev_options.rev): | |||||
logger.info( | |||||
'Updating %s %s%s', | |||||
display_path(dest), | |||||
self.repo_name, | |||||
rev_display, | |||||
) | |||||
self.update(dest, rev_options) | |||||
else: | |||||
logger.info( | |||||
'Skipping because already up-to-date.') | |||||
else: | |||||
logger.warning( | |||||
'%s %s in %s exists with URL %s', | |||||
self.name, | |||||
self.repo_name, | |||||
display_path(dest), | |||||
existing_url, | |||||
) | |||||
prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', | |||||
('s', 'i', 'w', 'b')) | |||||
else: | |||||
logger.warning( | |||||
'Directory %s already exists, and is not a %s %s.', | |||||
dest, | |||||
self.name, | |||||
self.repo_name, | |||||
) | |||||
prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) | |||||
if prompt: | |||||
logger.warning( | |||||
'The plan is to install the %s repository %s', | |||||
self.name, | |||||
url, | |||||
) | |||||
response = ask_path_exists('What to do? %s' % prompt[0], | |||||
prompt[1]) | |||||
if response == 's': | |||||
logger.info( | |||||
'Switching %s %s to %s%s', | |||||
self.repo_name, | |||||
display_path(dest), | |||||
url, | |||||
rev_display, | |||||
) | |||||
self.switch(dest, url, rev_options) | |||||
elif response == 'i': | |||||
# do nothing | |||||
pass | |||||
elif response == 'w': | |||||
logger.warning('Deleting %s', display_path(dest)) | |||||
rmtree(dest) | |||||
checkout = True | |||||
elif response == 'b': | |||||
dest_dir = backup_dir(dest) | |||||
logger.warning( | |||||
'Backing up %s to %s', display_path(dest), dest_dir, | |||||
) | |||||
shutil.move(dest, dest_dir) | |||||
checkout = True | |||||
elif response == 'a': | |||||
sys.exit(-1) | |||||
return checkout | |||||
def unpack(self, location): | |||||
""" | |||||
Clean up current location and download the url repository | |||||
(and vcs infos) into location | |||||
""" | |||||
if os.path.exists(location): | |||||
rmtree(location) | |||||
self.obtain(location) | |||||
def get_src_requirement(self, dist, location): | |||||
""" | |||||
Return a string representing the requirement needed to | |||||
redownload the files currently present in location, something | |||||
like: | |||||
{repository_url}@{revision}#egg={project_name}-{version_identifier} | |||||
""" | |||||
raise NotImplementedError | |||||
def get_url(self, location): | |||||
""" | |||||
Return the url used at location | |||||
Used in get_info or check_destination | |||||
""" | |||||
raise NotImplementedError | |||||
def get_revision(self, location): | |||||
""" | |||||
Return the current commit id of the files at the given location. | |||||
""" | |||||
raise NotImplementedError | |||||
def run_command(self, cmd, show_stdout=True, cwd=None, | |||||
on_returncode='raise', | |||||
command_desc=None, | |||||
extra_environ=None, spinner=None): | |||||
""" | |||||
Run a VCS subcommand | |||||
This is simply a wrapper around call_subprocess that adds the VCS | |||||
command name, and checks that the VCS is available | |||||
""" | |||||
cmd = [self.name] + cmd | |||||
try: | |||||
return call_subprocess(cmd, show_stdout, cwd, | |||||
on_returncode, | |||||
command_desc, extra_environ, | |||||
unset_environ=self.unset_environ, | |||||
spinner=spinner) | |||||
except OSError as e: | |||||
# errno.ENOENT = no such file or directory | |||||
# In other words, the VCS executable isn't available | |||||
if e.errno == errno.ENOENT: | |||||
raise BadCommand( | |||||
'Cannot find command %r - do you have ' | |||||
'%r installed and in your ' | |||||
'PATH?' % (self.name, self.name)) | |||||
else: | |||||
raise # re-raise exception if a different error occurred | |||||
@classmethod | |||||
def controls_location(cls, location): | |||||
""" | |||||
Check if a location is controlled by the vcs. | |||||
It is meant to be overridden to implement smarter detection | |||||
mechanisms for specific vcs. | |||||
""" | |||||
logger.debug('Checking in %s for %s (%s)...', | |||||
location, cls.dirname, cls.name) | |||||
path = os.path.join(location, cls.dirname) | |||||
return os.path.exists(path) | |||||
def get_src_requirement(dist, location): | |||||
version_control = vcs.get_backend_from_location(location) | |||||
if version_control: | |||||
try: | |||||
return version_control().get_src_requirement(dist, | |||||
location) | |||||
except BadCommand: | |||||
logger.warning( | |||||
'cannot determine version of editable source in %s ' | |||||
'(%s command not found in path)', | |||||
location, | |||||
version_control.name, | |||||
) | |||||
return dist.as_requirement() | |||||
logger.warning( | |||||
'cannot determine version of editable source in %s (is not SVN ' | |||||
'checkout, Git clone, Mercurial clone or Bazaar branch)', | |||||
location, | |||||
) | |||||
return dist.as_requirement() |
from __future__ import absolute_import | |||||
import logging | |||||
import os | |||||
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
from pip._internal.download import path_to_url | |||||
from pip._internal.utils.misc import display_path, rmtree | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.vcs import VersionControl, vcs | |||||
logger = logging.getLogger(__name__) | |||||
class Bazaar(VersionControl): | |||||
name = 'bzr' | |||||
dirname = '.bzr' | |||||
repo_name = 'branch' | |||||
schemes = ( | |||||
'bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp', | |||||
'bzr+lp', | |||||
) | |||||
def __init__(self, url=None, *args, **kwargs): | |||||
super(Bazaar, self).__init__(url, *args, **kwargs) | |||||
# This is only needed for python <2.7.5 | |||||
# Register lp but do not expose as a scheme to support bzr+lp. | |||||
if getattr(urllib_parse, 'uses_fragment', None): | |||||
urllib_parse.uses_fragment.extend(['lp']) | |||||
def get_base_rev_args(self, rev): | |||||
return ['-r', rev] | |||||
def export(self, location): | |||||
""" | |||||
Export the Bazaar repository at the url to the destination location | |||||
""" | |||||
# Remove the location to make sure Bazaar can export it correctly | |||||
if os.path.exists(location): | |||||
rmtree(location) | |||||
with TempDirectory(kind="export") as temp_dir: | |||||
self.unpack(temp_dir.path) | |||||
self.run_command( | |||||
['export', location], | |||||
cwd=temp_dir.path, show_stdout=False, | |||||
) | |||||
def switch(self, dest, url, rev_options): | |||||
self.run_command(['switch', url], cwd=dest) | |||||
def update(self, dest, rev_options): | |||||
cmd_args = ['pull', '-q'] + rev_options.to_args() | |||||
self.run_command(cmd_args, cwd=dest) | |||||
def obtain(self, dest): | |||||
url, rev = self.get_url_rev() | |||||
rev_options = self.make_rev_options(rev) | |||||
if self.check_destination(dest, url, rev_options): | |||||
rev_display = rev_options.to_display() | |||||
logger.info( | |||||
'Checking out %s%s to %s', | |||||
url, | |||||
rev_display, | |||||
display_path(dest), | |||||
) | |||||
cmd_args = ['branch', '-q'] + rev_options.to_args() + [url, dest] | |||||
self.run_command(cmd_args) | |||||
def get_url_rev(self): | |||||
# hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it | |||||
url, rev = super(Bazaar, self).get_url_rev() | |||||
if url.startswith('ssh://'): | |||||
url = 'bzr+' + url | |||||
return url, rev | |||||
def get_url(self, location): | |||||
urls = self.run_command(['info'], show_stdout=False, cwd=location) | |||||
for line in urls.splitlines(): | |||||
line = line.strip() | |||||
for x in ('checkout of branch: ', | |||||
'parent branch: '): | |||||
if line.startswith(x): | |||||
repo = line.split(x)[1] | |||||
if self._is_local_repository(repo): | |||||
return path_to_url(repo) | |||||
return repo | |||||
return None | |||||
def get_revision(self, location): | |||||
revision = self.run_command( | |||||
['revno'], show_stdout=False, cwd=location, | |||||
) | |||||
return revision.splitlines()[-1] | |||||
def get_src_requirement(self, dist, location): | |||||
repo = self.get_url(location) | |||||
if not repo: | |||||
return None | |||||
if not repo.lower().startswith('bzr:'): | |||||
repo = 'bzr+' + repo | |||||
egg_project_name = dist.egg_name().split('-', 1)[0] | |||||
current_rev = self.get_revision(location) | |||||
return '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) | |||||
def is_commit_id_equal(self, dest, name): | |||||
"""Always assume the versions don't match""" | |||||
return False | |||||
vcs.register(Bazaar) |
from __future__ import absolute_import | |||||
import logging | |||||
import os.path | |||||
import re | |||||
from pip._vendor.packaging.version import parse as parse_version | |||||
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
from pip._vendor.six.moves.urllib import request as urllib_request | |||||
from pip._internal.compat import samefile | |||||
from pip._internal.exceptions import BadCommand | |||||
from pip._internal.utils.misc import display_path | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.vcs import VersionControl, vcs | |||||
urlsplit = urllib_parse.urlsplit | |||||
urlunsplit = urllib_parse.urlunsplit | |||||
logger = logging.getLogger(__name__) | |||||
HASH_REGEX = re.compile('[a-fA-F0-9]{40}') | |||||
def looks_like_hash(sha): | |||||
return bool(HASH_REGEX.match(sha)) | |||||
class Git(VersionControl): | |||||
name = 'git' | |||||
dirname = '.git' | |||||
repo_name = 'clone' | |||||
schemes = ( | |||||
'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', | |||||
) | |||||
# Prevent the user's environment variables from interfering with pip: | |||||
# https://github.com/pypa/pip/issues/1130 | |||||
unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') | |||||
default_arg_rev = 'HEAD' | |||||
def __init__(self, url=None, *args, **kwargs): | |||||
# Works around an apparent Git bug | |||||
# (see http://article.gmane.org/gmane.comp.version-control.git/146500) | |||||
if url: | |||||
scheme, netloc, path, query, fragment = urlsplit(url) | |||||
if scheme.endswith('file'): | |||||
initial_slashes = path[:-len(path.lstrip('/'))] | |||||
newpath = ( | |||||
initial_slashes + | |||||
urllib_request.url2pathname(path) | |||||
.replace('\\', '/').lstrip('/') | |||||
) | |||||
url = urlunsplit((scheme, netloc, newpath, query, fragment)) | |||||
after_plus = scheme.find('+') + 1 | |||||
url = scheme[:after_plus] + urlunsplit( | |||||
(scheme[after_plus:], netloc, newpath, query, fragment), | |||||
) | |||||
super(Git, self).__init__(url, *args, **kwargs) | |||||
def get_base_rev_args(self, rev): | |||||
return [rev] | |||||
def get_git_version(self): | |||||
VERSION_PFX = 'git version ' | |||||
version = self.run_command(['version'], show_stdout=False) | |||||
if version.startswith(VERSION_PFX): | |||||
version = version[len(VERSION_PFX):].split()[0] | |||||
else: | |||||
version = '' | |||||
# get first 3 positions of the git version becasue | |||||
# on windows it is x.y.z.windows.t, and this parses as | |||||
# LegacyVersion which always smaller than a Version. | |||||
version = '.'.join(version.split('.')[:3]) | |||||
return parse_version(version) | |||||
def export(self, location): | |||||
"""Export the Git repository at the url to the destination location""" | |||||
if not location.endswith('/'): | |||||
location = location + '/' | |||||
with TempDirectory(kind="export") as temp_dir: | |||||
self.unpack(temp_dir.path) | |||||
self.run_command( | |||||
['checkout-index', '-a', '-f', '--prefix', location], | |||||
show_stdout=False, cwd=temp_dir.path | |||||
) | |||||
def get_revision_sha(self, dest, rev): | |||||
""" | |||||
Return a commit hash for the given revision if it names a remote | |||||
branch or tag. Otherwise, return None. | |||||
Args: | |||||
dest: the repository directory. | |||||
rev: the revision name. | |||||
""" | |||||
# Pass rev to pre-filter the list. | |||||
output = self.run_command(['show-ref', rev], cwd=dest, | |||||
show_stdout=False, on_returncode='ignore') | |||||
refs = {} | |||||
for line in output.strip().splitlines(): | |||||
try: | |||||
sha, ref = line.split() | |||||
except ValueError: | |||||
# Include the offending line to simplify troubleshooting if | |||||
# this error ever occurs. | |||||
raise ValueError('unexpected show-ref line: {!r}'.format(line)) | |||||
refs[ref] = sha | |||||
branch_ref = 'refs/remotes/origin/{}'.format(rev) | |||||
tag_ref = 'refs/tags/{}'.format(rev) | |||||
return refs.get(branch_ref) or refs.get(tag_ref) | |||||
def check_rev_options(self, dest, rev_options): | |||||
"""Check the revision options before checkout. | |||||
Returns a new RevOptions object for the SHA1 of the branch or tag | |||||
if found. | |||||
Args: | |||||
rev_options: a RevOptions object. | |||||
""" | |||||
rev = rev_options.arg_rev | |||||
sha = self.get_revision_sha(dest, rev) | |||||
if sha is not None: | |||||
return rev_options.make_new(sha) | |||||
# Do not show a warning for the common case of something that has | |||||
# the form of a Git commit hash. | |||||
if not looks_like_hash(rev): | |||||
logger.warning( | |||||
"Did not find branch or tag '%s', assuming revision or ref.", | |||||
rev, | |||||
) | |||||
return rev_options | |||||
def is_commit_id_equal(self, dest, name): | |||||
""" | |||||
Return whether the current commit hash equals the given name. | |||||
Args: | |||||
dest: the repository directory. | |||||
name: a string name. | |||||
""" | |||||
if not name: | |||||
# Then avoid an unnecessary subprocess call. | |||||
return False | |||||
return self.get_revision(dest) == name | |||||
def switch(self, dest, url, rev_options): | |||||
self.run_command(['config', 'remote.origin.url', url], cwd=dest) | |||||
cmd_args = ['checkout', '-q'] + rev_options.to_args() | |||||
self.run_command(cmd_args, cwd=dest) | |||||
self.update_submodules(dest) | |||||
def update(self, dest, rev_options): | |||||
# First fetch changes from the default remote | |||||
if self.get_git_version() >= parse_version('1.9.0'): | |||||
# fetch tags in addition to everything else | |||||
self.run_command(['fetch', '-q', '--tags'], cwd=dest) | |||||
else: | |||||
self.run_command(['fetch', '-q'], cwd=dest) | |||||
# Then reset to wanted revision (maybe even origin/master) | |||||
rev_options = self.check_rev_options(dest, rev_options) | |||||
cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args() | |||||
self.run_command(cmd_args, cwd=dest) | |||||
#: update submodules | |||||
self.update_submodules(dest) | |||||
def obtain(self, dest): | |||||
url, rev = self.get_url_rev() | |||||
rev_options = self.make_rev_options(rev) | |||||
if self.check_destination(dest, url, rev_options): | |||||
rev_display = rev_options.to_display() | |||||
logger.info( | |||||
'Cloning %s%s to %s', url, rev_display, display_path(dest), | |||||
) | |||||
self.run_command(['clone', '-q', url, dest]) | |||||
if rev: | |||||
rev_options = self.check_rev_options(dest, rev_options) | |||||
# Only do a checkout if the current commit id doesn't match | |||||
# the requested revision. | |||||
if not self.is_commit_id_equal(dest, rev_options.rev): | |||||
rev = rev_options.rev | |||||
# Only fetch the revision if it's a ref | |||||
if rev.startswith('refs/'): | |||||
self.run_command( | |||||
['fetch', '-q', url] + rev_options.to_args(), | |||||
cwd=dest, | |||||
) | |||||
# Change the revision to the SHA of the ref we fetched | |||||
rev = 'FETCH_HEAD' | |||||
self.run_command(['checkout', '-q', rev], cwd=dest) | |||||
#: repo may contain submodules | |||||
self.update_submodules(dest) | |||||
def get_url(self, location): | |||||
"""Return URL of the first remote encountered.""" | |||||
remotes = self.run_command( | |||||
['config', '--get-regexp', r'remote\..*\.url'], | |||||
show_stdout=False, cwd=location, | |||||
) | |||||
remotes = remotes.splitlines() | |||||
found_remote = remotes[0] | |||||
for remote in remotes: | |||||
if remote.startswith('remote.origin.url '): | |||||
found_remote = remote | |||||
break | |||||
url = found_remote.split(' ')[1] | |||||
return url.strip() | |||||
def get_revision(self, location): | |||||
current_rev = self.run_command( | |||||
['rev-parse', 'HEAD'], show_stdout=False, cwd=location, | |||||
) | |||||
return current_rev.strip() | |||||
def _get_subdirectory(self, location): | |||||
"""Return the relative path of setup.py to the git repo root.""" | |||||
# find the repo root | |||||
git_dir = self.run_command(['rev-parse', '--git-dir'], | |||||
show_stdout=False, cwd=location).strip() | |||||
if not os.path.isabs(git_dir): | |||||
git_dir = os.path.join(location, git_dir) | |||||
root_dir = os.path.join(git_dir, '..') | |||||
# find setup.py | |||||
orig_location = location | |||||
while not os.path.exists(os.path.join(location, 'setup.py')): | |||||
last_location = location | |||||
location = os.path.dirname(location) | |||||
if location == last_location: | |||||
# We've traversed up to the root of the filesystem without | |||||
# finding setup.py | |||||
logger.warning( | |||||
"Could not find setup.py for directory %s (tried all " | |||||
"parent directories)", | |||||
orig_location, | |||||
) | |||||
return None | |||||
# relative path of setup.py to repo root | |||||
if samefile(root_dir, location): | |||||
return None | |||||
return os.path.relpath(location, root_dir) | |||||
def get_src_requirement(self, dist, location): | |||||
repo = self.get_url(location) | |||||
if not repo.lower().startswith('git:'): | |||||
repo = 'git+' + repo | |||||
egg_project_name = dist.egg_name().split('-', 1)[0] | |||||
if not repo: | |||||
return None | |||||
current_rev = self.get_revision(location) | |||||
req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) | |||||
subdirectory = self._get_subdirectory(location) | |||||
if subdirectory: | |||||
req += '&subdirectory=' + subdirectory | |||||
return req | |||||
def get_url_rev(self): | |||||
""" | |||||
Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. | |||||
That's required because although they use SSH they sometimes doesn't | |||||
work with a ssh:// scheme (e.g. Github). But we need a scheme for | |||||
parsing. Hence we remove it again afterwards and return it as a stub. | |||||
""" | |||||
if '://' not in self.url: | |||||
assert 'file:' not in self.url | |||||
self.url = self.url.replace('git+', 'git+ssh://') | |||||
url, rev = super(Git, self).get_url_rev() | |||||
url = url.replace('ssh://', '') | |||||
else: | |||||
url, rev = super(Git, self).get_url_rev() | |||||
return url, rev | |||||
def update_submodules(self, location): | |||||
if not os.path.exists(os.path.join(location, '.gitmodules')): | |||||
return | |||||
self.run_command( | |||||
['submodule', 'update', '--init', '--recursive', '-q'], | |||||
cwd=location, | |||||
) | |||||
@classmethod | |||||
def controls_location(cls, location): | |||||
if super(Git, cls).controls_location(location): | |||||
return True | |||||
try: | |||||
r = cls().run_command(['rev-parse'], | |||||
cwd=location, | |||||
show_stdout=False, | |||||
on_returncode='ignore') | |||||
return not r | |||||
except BadCommand: | |||||
logger.debug("could not determine if %s is under git control " | |||||
"because git is not available", location) | |||||
return False | |||||
vcs.register(Git) |
from __future__ import absolute_import | |||||
import logging | |||||
import os | |||||
from pip._vendor.six.moves import configparser | |||||
from pip._internal.download import path_to_url | |||||
from pip._internal.utils.misc import display_path | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.vcs import VersionControl, vcs | |||||
logger = logging.getLogger(__name__) | |||||
class Mercurial(VersionControl): | |||||
name = 'hg' | |||||
dirname = '.hg' | |||||
repo_name = 'clone' | |||||
schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http') | |||||
def get_base_rev_args(self, rev): | |||||
return [rev] | |||||
def export(self, location): | |||||
"""Export the Hg repository at the url to the destination location""" | |||||
with TempDirectory(kind="export") as temp_dir: | |||||
self.unpack(temp_dir.path) | |||||
self.run_command( | |||||
['archive', location], show_stdout=False, cwd=temp_dir.path | |||||
) | |||||
def switch(self, dest, url, rev_options): | |||||
repo_config = os.path.join(dest, self.dirname, 'hgrc') | |||||
config = configparser.SafeConfigParser() | |||||
try: | |||||
config.read(repo_config) | |||||
config.set('paths', 'default', url) | |||||
with open(repo_config, 'w') as config_file: | |||||
config.write(config_file) | |||||
except (OSError, configparser.NoSectionError) as exc: | |||||
logger.warning( | |||||
'Could not switch Mercurial repository to %s: %s', url, exc, | |||||
) | |||||
else: | |||||
cmd_args = ['update', '-q'] + rev_options.to_args() | |||||
self.run_command(cmd_args, cwd=dest) | |||||
def update(self, dest, rev_options): | |||||
self.run_command(['pull', '-q'], cwd=dest) | |||||
cmd_args = ['update', '-q'] + rev_options.to_args() | |||||
self.run_command(cmd_args, cwd=dest) | |||||
def obtain(self, dest): | |||||
url, rev = self.get_url_rev() | |||||
rev_options = self.make_rev_options(rev) | |||||
if self.check_destination(dest, url, rev_options): | |||||
rev_display = rev_options.to_display() | |||||
logger.info( | |||||
'Cloning hg %s%s to %s', | |||||
url, | |||||
rev_display, | |||||
display_path(dest), | |||||
) | |||||
self.run_command(['clone', '--noupdate', '-q', url, dest]) | |||||
cmd_args = ['update', '-q'] + rev_options.to_args() | |||||
self.run_command(cmd_args, cwd=dest) | |||||
def get_url(self, location): | |||||
url = self.run_command( | |||||
['showconfig', 'paths.default'], | |||||
show_stdout=False, cwd=location).strip() | |||||
if self._is_local_repository(url): | |||||
url = path_to_url(url) | |||||
return url.strip() | |||||
def get_revision(self, location): | |||||
current_revision = self.run_command( | |||||
['parents', '--template={rev}'], | |||||
show_stdout=False, cwd=location).strip() | |||||
return current_revision | |||||
def get_revision_hash(self, location): | |||||
current_rev_hash = self.run_command( | |||||
['parents', '--template={node}'], | |||||
show_stdout=False, cwd=location).strip() | |||||
return current_rev_hash | |||||
def get_src_requirement(self, dist, location): | |||||
repo = self.get_url(location) | |||||
if not repo.lower().startswith('hg:'): | |||||
repo = 'hg+' + repo | |||||
egg_project_name = dist.egg_name().split('-', 1)[0] | |||||
if not repo: | |||||
return None | |||||
current_rev_hash = self.get_revision_hash(location) | |||||
return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name) | |||||
def is_commit_id_equal(self, dest, name): | |||||
"""Always assume the versions don't match""" | |||||
return False | |||||
vcs.register(Mercurial) |
from __future__ import absolute_import | |||||
import logging | |||||
import os | |||||
import re | |||||
from pip._vendor.six.moves.urllib import parse as urllib_parse | |||||
from pip._internal.index import Link | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.utils.misc import display_path, rmtree | |||||
from pip._internal.vcs import VersionControl, vcs | |||||
_svn_xml_url_re = re.compile('url="([^"]+)"') | |||||
_svn_rev_re = re.compile(r'committed-rev="(\d+)"') | |||||
_svn_url_re = re.compile(r'URL: (.+)') | |||||
_svn_revision_re = re.compile(r'Revision: (.+)') | |||||
_svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') | |||||
_svn_info_xml_url_re = re.compile(r'<url>(.*)</url>') | |||||
logger = logging.getLogger(__name__) | |||||
class Subversion(VersionControl): | |||||
name = 'svn' | |||||
dirname = '.svn' | |||||
repo_name = 'checkout' | |||||
schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https', 'svn+svn') | |||||
def get_base_rev_args(self, rev): | |||||
return ['-r', rev] | |||||
def get_info(self, location): | |||||
"""Returns (url, revision), where both are strings""" | |||||
assert not location.rstrip('/').endswith(self.dirname), \ | |||||
'Bad directory: %s' % location | |||||
output = self.run_command( | |||||
['info', location], | |||||
show_stdout=False, | |||||
extra_environ={'LANG': 'C'}, | |||||
) | |||||
match = _svn_url_re.search(output) | |||||
if not match: | |||||
logger.warning( | |||||
'Cannot determine URL of svn checkout %s', | |||||
display_path(location), | |||||
) | |||||
logger.debug('Output that cannot be parsed: \n%s', output) | |||||
return None, None | |||||
url = match.group(1).strip() | |||||
match = _svn_revision_re.search(output) | |||||
if not match: | |||||
logger.warning( | |||||
'Cannot determine revision of svn checkout %s', | |||||
display_path(location), | |||||
) | |||||
logger.debug('Output that cannot be parsed: \n%s', output) | |||||
return url, None | |||||
return url, match.group(1) | |||||
def export(self, location): | |||||
"""Export the svn repository at the url to the destination location""" | |||||
url, rev = self.get_url_rev() | |||||
rev_options = get_rev_options(self, url, rev) | |||||
url = self.remove_auth_from_url(url) | |||||
logger.info('Exporting svn repository %s to %s', url, location) | |||||
with indent_log(): | |||||
if os.path.exists(location): | |||||
# Subversion doesn't like to check out over an existing | |||||
# directory --force fixes this, but was only added in svn 1.5 | |||||
rmtree(location) | |||||
cmd_args = ['export'] + rev_options.to_args() + [url, location] | |||||
self.run_command(cmd_args, show_stdout=False) | |||||
def switch(self, dest, url, rev_options): | |||||
cmd_args = ['switch'] + rev_options.to_args() + [url, dest] | |||||
self.run_command(cmd_args) | |||||
def update(self, dest, rev_options): | |||||
cmd_args = ['update'] + rev_options.to_args() + [dest] | |||||
self.run_command(cmd_args) | |||||
def obtain(self, dest): | |||||
url, rev = self.get_url_rev() | |||||
rev_options = get_rev_options(self, url, rev) | |||||
url = self.remove_auth_from_url(url) | |||||
if self.check_destination(dest, url, rev_options): | |||||
rev_display = rev_options.to_display() | |||||
logger.info( | |||||
'Checking out %s%s to %s', | |||||
url, | |||||
rev_display, | |||||
display_path(dest), | |||||
) | |||||
cmd_args = ['checkout', '-q'] + rev_options.to_args() + [url, dest] | |||||
self.run_command(cmd_args) | |||||
def get_location(self, dist, dependency_links): | |||||
for url in dependency_links: | |||||
egg_fragment = Link(url).egg_fragment | |||||
if not egg_fragment: | |||||
continue | |||||
if '-' in egg_fragment: | |||||
# FIXME: will this work when a package has - in the name? | |||||
key = '-'.join(egg_fragment.split('-')[:-1]).lower() | |||||
else: | |||||
key = egg_fragment | |||||
if key == dist.key: | |||||
return url.split('#', 1)[0] | |||||
return None | |||||
def get_revision(self, location): | |||||
""" | |||||
Return the maximum revision for all files under a given location | |||||
""" | |||||
# Note: taken from setuptools.command.egg_info | |||||
revision = 0 | |||||
for base, dirs, files in os.walk(location): | |||||
if self.dirname not in dirs: | |||||
dirs[:] = [] | |||||
continue # no sense walking uncontrolled subdirs | |||||
dirs.remove(self.dirname) | |||||
entries_fn = os.path.join(base, self.dirname, 'entries') | |||||
if not os.path.exists(entries_fn): | |||||
# FIXME: should we warn? | |||||
continue | |||||
dirurl, localrev = self._get_svn_url_rev(base) | |||||
if base == location: | |||||
base = dirurl + '/' # save the root url | |||||
elif not dirurl or not dirurl.startswith(base): | |||||
dirs[:] = [] | |||||
continue # not part of the same svn tree, skip it | |||||
revision = max(revision, localrev) | |||||
return revision | |||||
def get_url_rev(self): | |||||
# hotfix the URL scheme after removing svn+ from svn+ssh:// readd it | |||||
url, rev = super(Subversion, self).get_url_rev() | |||||
if url.startswith('ssh://'): | |||||
url = 'svn+' + url | |||||
return url, rev | |||||
def get_url(self, location): | |||||
# In cases where the source is in a subdirectory, not alongside | |||||
# setup.py we have to look up in the location until we find a real | |||||
# setup.py | |||||
orig_location = location | |||||
while not os.path.exists(os.path.join(location, 'setup.py')): | |||||
last_location = location | |||||
location = os.path.dirname(location) | |||||
if location == last_location: | |||||
# We've traversed up to the root of the filesystem without | |||||
# finding setup.py | |||||
logger.warning( | |||||
"Could not find setup.py for directory %s (tried all " | |||||
"parent directories)", | |||||
orig_location, | |||||
) | |||||
return None | |||||
return self._get_svn_url_rev(location)[0] | |||||
def _get_svn_url_rev(self, location): | |||||
from pip._internal.exceptions import InstallationError | |||||
entries_path = os.path.join(location, self.dirname, 'entries') | |||||
if os.path.exists(entries_path): | |||||
with open(entries_path) as f: | |||||
data = f.read() | |||||
else: # subversion >= 1.7 does not have the 'entries' file | |||||
data = '' | |||||
if (data.startswith('8') or | |||||
data.startswith('9') or | |||||
data.startswith('10')): | |||||
data = list(map(str.splitlines, data.split('\n\x0c\n'))) | |||||
del data[0][0] # get rid of the '8' | |||||
url = data[0][3] | |||||
revs = [int(d[9]) for d in data if len(d) > 9 and d[9]] + [0] | |||||
elif data.startswith('<?xml'): | |||||
match = _svn_xml_url_re.search(data) | |||||
if not match: | |||||
raise ValueError('Badly formatted data: %r' % data) | |||||
url = match.group(1) # get repository URL | |||||
revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)] + [0] | |||||
else: | |||||
try: | |||||
# subversion >= 1.7 | |||||
xml = self.run_command( | |||||
['info', '--xml', location], | |||||
show_stdout=False, | |||||
) | |||||
url = _svn_info_xml_url_re.search(xml).group(1) | |||||
revs = [ | |||||
int(m.group(1)) for m in _svn_info_xml_rev_re.finditer(xml) | |||||
] | |||||
except InstallationError: | |||||
url, revs = None, [] | |||||
if revs: | |||||
rev = max(revs) | |||||
else: | |||||
rev = 0 | |||||
return url, rev | |||||
def get_src_requirement(self, dist, location): | |||||
repo = self.get_url(location) | |||||
if repo is None: | |||||
return None | |||||
# FIXME: why not project name? | |||||
egg_project_name = dist.egg_name().split('-', 1)[0] | |||||
rev = self.get_revision(location) | |||||
return 'svn+%s@%s#egg=%s' % (repo, rev, egg_project_name) | |||||
def is_commit_id_equal(self, dest, name): | |||||
"""Always assume the versions don't match""" | |||||
return False | |||||
@staticmethod | |||||
def remove_auth_from_url(url): | |||||
# Return a copy of url with 'username:password@' removed. | |||||
# username/pass params are passed to subversion through flags | |||||
# and are not recognized in the url. | |||||
# parsed url | |||||
purl = urllib_parse.urlsplit(url) | |||||
stripped_netloc = \ | |||||
purl.netloc.split('@')[-1] | |||||
# stripped url | |||||
url_pieces = ( | |||||
purl.scheme, stripped_netloc, purl.path, purl.query, purl.fragment | |||||
) | |||||
surl = urllib_parse.urlunsplit(url_pieces) | |||||
return surl | |||||
def get_rev_options(vcs, url, rev): | |||||
""" | |||||
Return a RevOptions object. | |||||
""" | |||||
r = urllib_parse.urlsplit(url) | |||||
if hasattr(r, 'username'): | |||||
# >= Python-2.5 | |||||
username, password = r.username, r.password | |||||
else: | |||||
netloc = r[1] | |||||
if '@' in netloc: | |||||
auth = netloc.split('@')[0] | |||||
if ':' in auth: | |||||
username, password = auth.split(':', 1) | |||||
else: | |||||
username, password = auth, None | |||||
else: | |||||
username, password = None, None | |||||
extra_args = [] | |||||
if username: | |||||
extra_args += ['--username', username] | |||||
if password: | |||||
extra_args += ['--password', password] | |||||
return vcs.make_rev_options(rev, extra_args=extra_args) | |||||
vcs.register(Subversion) |
""" | |||||
Support for installing and building the "wheel" binary package format. | |||||
""" | |||||
from __future__ import absolute_import | |||||
import collections | |||||
import compileall | |||||
import copy | |||||
import csv | |||||
import hashlib | |||||
import logging | |||||
import os.path | |||||
import re | |||||
import shutil | |||||
import stat | |||||
import sys | |||||
import warnings | |||||
from base64 import urlsafe_b64encode | |||||
from email.parser import Parser | |||||
from pip._vendor import pkg_resources | |||||
from pip._vendor.distlib.scripts import ScriptMaker | |||||
from pip._vendor.packaging.utils import canonicalize_name | |||||
from pip._vendor.six import StringIO | |||||
from pip._internal import pep425tags | |||||
from pip._internal.build_env import BuildEnvironment | |||||
from pip._internal.download import path_to_url, unpack_url | |||||
from pip._internal.exceptions import ( | |||||
InstallationError, InvalidWheelFilename, UnsupportedWheel, | |||||
) | |||||
from pip._internal.locations import ( | |||||
PIP_DELETE_MARKER_FILENAME, distutils_scheme, | |||||
) | |||||
from pip._internal.utils.logging import indent_log | |||||
from pip._internal.utils.misc import ( | |||||
call_subprocess, captured_stdout, ensure_dir, read_chunks, | |||||
) | |||||
from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM | |||||
from pip._internal.utils.temp_dir import TempDirectory | |||||
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |||||
from pip._internal.utils.ui import open_spinner | |||||
if MYPY_CHECK_RUNNING: | |||||
from typing import Dict, List, Optional | |||||
wheel_ext = '.whl' | |||||
VERSION_COMPATIBLE = (1, 0) | |||||
logger = logging.getLogger(__name__) | |||||
def rehash(path, algo='sha256', blocksize=1 << 20): | |||||
"""Return (hash, length) for path using hashlib.new(algo)""" | |||||
h = hashlib.new(algo) | |||||
length = 0 | |||||
with open(path, 'rb') as f: | |||||
for block in read_chunks(f, size=blocksize): | |||||
length += len(block) | |||||
h.update(block) | |||||
digest = 'sha256=' + urlsafe_b64encode( | |||||
h.digest() | |||||
).decode('latin1').rstrip('=') | |||||
return (digest, length) | |||||
def open_for_csv(name, mode): | |||||
if sys.version_info[0] < 3: | |||||
nl = {} | |||||
bin = 'b' | |||||
else: | |||||
nl = {'newline': ''} | |||||
bin = '' | |||||
return open(name, mode + bin, **nl) | |||||
def fix_script(path): | |||||
"""Replace #!python with #!/path/to/python | |||||
Return True if file was changed.""" | |||||
# XXX RECORD hashes will need to be updated | |||||
if os.path.isfile(path): | |||||
with open(path, 'rb') as script: | |||||
firstline = script.readline() | |||||
if not firstline.startswith(b'#!python'): | |||||
return False | |||||
exename = sys.executable.encode(sys.getfilesystemencoding()) | |||||
firstline = b'#!' + exename + os.linesep.encode("ascii") | |||||
rest = script.read() | |||||
with open(path, 'wb') as script: | |||||
script.write(firstline) | |||||
script.write(rest) | |||||
return True | |||||
dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>.+?))?) | |||||
\.dist-info$""", re.VERBOSE) | |||||
def root_is_purelib(name, wheeldir): | |||||
""" | |||||
Return True if the extracted wheel in wheeldir should go into purelib. | |||||
""" | |||||
name_folded = name.replace("-", "_") | |||||
for item in os.listdir(wheeldir): | |||||
match = dist_info_re.match(item) | |||||
if match and match.group('name') == name_folded: | |||||
with open(os.path.join(wheeldir, item, 'WHEEL')) as wheel: | |||||
for line in wheel: | |||||
line = line.lower().rstrip() | |||||
if line == "root-is-purelib: true": | |||||
return True | |||||
return False | |||||
def get_entrypoints(filename): | |||||
if not os.path.exists(filename): | |||||
return {}, {} | |||||
# This is done because you can pass a string to entry_points wrappers which | |||||
# means that they may or may not be valid INI files. The attempt here is to | |||||
# strip leading and trailing whitespace in order to make them valid INI | |||||
# files. | |||||
with open(filename) as fp: | |||||
data = StringIO() | |||||
for line in fp: | |||||
data.write(line.strip()) | |||||
data.write("\n") | |||||
data.seek(0) | |||||
# get the entry points and then the script names | |||||
entry_points = pkg_resources.EntryPoint.parse_map(data) | |||||
console = entry_points.get('console_scripts', {}) | |||||
gui = entry_points.get('gui_scripts', {}) | |||||
def _split_ep(s): | |||||
"""get the string representation of EntryPoint, remove space and split | |||||
on '='""" | |||||
return str(s).replace(" ", "").split("=") | |||||
# convert the EntryPoint objects into strings with module:function | |||||
console = dict(_split_ep(v) for v in console.values()) | |||||
gui = dict(_split_ep(v) for v in gui.values()) | |||||
return console, gui | |||||
def message_about_scripts_not_on_PATH(scripts): | |||||
# type: (List[str]) -> Optional[str] | |||||
"""Determine if any scripts are not on PATH and format a warning. | |||||
Returns a warning message if one or more scripts are not on PATH, | |||||
otherwise None. | |||||
""" | |||||
if not scripts: | |||||
return None | |||||
# Group scripts by the path they were installed in | |||||
grouped_by_dir = collections.defaultdict(set) # type: Dict[str, set] | |||||
for destfile in scripts: | |||||
parent_dir = os.path.dirname(destfile) | |||||
script_name = os.path.basename(destfile) | |||||
grouped_by_dir[parent_dir].add(script_name) | |||||
# We don't want to warn for directories that are on PATH. | |||||
not_warn_dirs = [ | |||||
os.path.normcase(i) for i in os.environ["PATH"].split(os.pathsep) | |||||
] | |||||
# If an executable sits with sys.executable, we don't warn for it. | |||||
# This covers the case of venv invocations without activating the venv. | |||||
not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) | |||||
warn_for = { | |||||
parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() | |||||
if os.path.normcase(parent_dir) not in not_warn_dirs | |||||
} | |||||
if not warn_for: | |||||
return None | |||||
# Format a message | |||||
msg_lines = [] | |||||
for parent_dir, scripts in warn_for.items(): | |||||
scripts = sorted(scripts) | |||||
if len(scripts) == 1: | |||||
start_text = "script {} is".format(scripts[0]) | |||||
else: | |||||
start_text = "scripts {} are".format( | |||||
", ".join(scripts[:-1]) + " and " + scripts[-1] | |||||
) | |||||
msg_lines.append( | |||||
"The {} installed in '{}' which is not on PATH." | |||||
.format(start_text, parent_dir) | |||||
) | |||||
last_line_fmt = ( | |||||
"Consider adding {} to PATH or, if you prefer " | |||||
"to suppress this warning, use --no-warn-script-location." | |||||
) | |||||
if len(msg_lines) == 1: | |||||
msg_lines.append(last_line_fmt.format("this directory")) | |||||
else: | |||||
msg_lines.append(last_line_fmt.format("these directories")) | |||||
# Returns the formatted multiline message | |||||
return "\n".join(msg_lines) | |||||
def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, | |||||
pycompile=True, scheme=None, isolated=False, prefix=None, | |||||
warn_script_location=True): | |||||
"""Install a wheel""" | |||||
if not scheme: | |||||
scheme = distutils_scheme( | |||||
name, user=user, home=home, root=root, isolated=isolated, | |||||
prefix=prefix, | |||||
) | |||||
if root_is_purelib(name, wheeldir): | |||||
lib_dir = scheme['purelib'] | |||||
else: | |||||
lib_dir = scheme['platlib'] | |||||
info_dir = [] | |||||
data_dirs = [] | |||||
source = wheeldir.rstrip(os.path.sep) + os.path.sep | |||||
# Record details of the files moved | |||||
# installed = files copied from the wheel to the destination | |||||
# changed = files changed while installing (scripts #! line typically) | |||||
# generated = files newly generated during the install (script wrappers) | |||||
installed = {} | |||||
changed = set() | |||||
generated = [] | |||||
# Compile all of the pyc files that we're going to be installing | |||||
if pycompile: | |||||
with captured_stdout() as stdout: | |||||
with warnings.catch_warnings(): | |||||
warnings.filterwarnings('ignore') | |||||
compileall.compile_dir(source, force=True, quiet=True) | |||||
logger.debug(stdout.getvalue()) | |||||
def normpath(src, p): | |||||
return os.path.relpath(src, p).replace(os.path.sep, '/') | |||||
def record_installed(srcfile, destfile, modified=False): | |||||
"""Map archive RECORD paths to installation RECORD paths.""" | |||||
oldpath = normpath(srcfile, wheeldir) | |||||
newpath = normpath(destfile, lib_dir) | |||||
installed[oldpath] = newpath | |||||
if modified: | |||||
changed.add(destfile) | |||||
def clobber(source, dest, is_base, fixer=None, filter=None): | |||||
ensure_dir(dest) # common for the 'include' path | |||||
for dir, subdirs, files in os.walk(source): | |||||
basedir = dir[len(source):].lstrip(os.path.sep) | |||||
destdir = os.path.join(dest, basedir) | |||||
if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'): | |||||
continue | |||||
for s in subdirs: | |||||
destsubdir = os.path.join(dest, basedir, s) | |||||
if is_base and basedir == '' and destsubdir.endswith('.data'): | |||||
data_dirs.append(s) | |||||
continue | |||||
elif (is_base and | |||||
s.endswith('.dist-info') and | |||||
canonicalize_name(s).startswith( | |||||
canonicalize_name(req.name))): | |||||
assert not info_dir, ('Multiple .dist-info directories: ' + | |||||
destsubdir + ', ' + | |||||
', '.join(info_dir)) | |||||
info_dir.append(destsubdir) | |||||
for f in files: | |||||
# Skip unwanted files | |||||
if filter and filter(f): | |||||
continue | |||||
srcfile = os.path.join(dir, f) | |||||
destfile = os.path.join(dest, basedir, f) | |||||
# directory creation is lazy and after the file filtering above | |||||
# to ensure we don't install empty dirs; empty dirs can't be | |||||
# uninstalled. | |||||
ensure_dir(destdir) | |||||
# We use copyfile (not move, copy, or copy2) to be extra sure | |||||
# that we are not moving directories over (copyfile fails for | |||||
# directories) as well as to ensure that we are not copying | |||||
# over any metadata because we want more control over what | |||||
# metadata we actually copy over. | |||||
shutil.copyfile(srcfile, destfile) | |||||
# Copy over the metadata for the file, currently this only | |||||
# includes the atime and mtime. | |||||
st = os.stat(srcfile) | |||||
if hasattr(os, "utime"): | |||||
os.utime(destfile, (st.st_atime, st.st_mtime)) | |||||
# If our file is executable, then make our destination file | |||||
# executable. | |||||
if os.access(srcfile, os.X_OK): | |||||
st = os.stat(srcfile) | |||||
permissions = ( | |||||
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | |||||
) | |||||
os.chmod(destfile, permissions) | |||||
changed = False | |||||
if fixer: | |||||
changed = fixer(destfile) | |||||
record_installed(srcfile, destfile, changed) | |||||
clobber(source, lib_dir, True) | |||||
assert info_dir, "%s .dist-info directory not found" % req | |||||
# Get the defined entry points | |||||
ep_file = os.path.join(info_dir[0], 'entry_points.txt') | |||||
console, gui = get_entrypoints(ep_file) | |||||
def is_entrypoint_wrapper(name): | |||||
# EP, EP.exe and EP-script.py are scripts generated for | |||||
# entry point EP by setuptools | |||||
if name.lower().endswith('.exe'): | |||||
matchname = name[:-4] | |||||
elif name.lower().endswith('-script.py'): | |||||
matchname = name[:-10] | |||||
elif name.lower().endswith(".pya"): | |||||
matchname = name[:-4] | |||||
else: | |||||
matchname = name | |||||
# Ignore setuptools-generated scripts | |||||
return (matchname in console or matchname in gui) | |||||
for datadir in data_dirs: | |||||
fixer = None | |||||
filter = None | |||||
for subdir in os.listdir(os.path.join(wheeldir, datadir)): | |||||
fixer = None | |||||
if subdir == 'scripts': | |||||
fixer = fix_script | |||||
filter = is_entrypoint_wrapper | |||||
source = os.path.join(wheeldir, datadir, subdir) | |||||
dest = scheme[subdir] | |||||
clobber(source, dest, False, fixer=fixer, filter=filter) | |||||
maker = ScriptMaker(None, scheme['scripts']) | |||||
# Ensure old scripts are overwritten. | |||||
# See https://github.com/pypa/pip/issues/1800 | |||||
maker.clobber = True | |||||
# Ensure we don't generate any variants for scripts because this is almost | |||||
# never what somebody wants. | |||||
# See https://bitbucket.org/pypa/distlib/issue/35/ | |||||
maker.variants = {''} | |||||
# This is required because otherwise distlib creates scripts that are not | |||||
# executable. | |||||
# See https://bitbucket.org/pypa/distlib/issue/32/ | |||||
maker.set_mode = True | |||||
# Simplify the script and fix the fact that the default script swallows | |||||
# every single stack trace. | |||||
# See https://bitbucket.org/pypa/distlib/issue/34/ | |||||
# See https://bitbucket.org/pypa/distlib/issue/33/ | |||||
def _get_script_text(entry): | |||||
if entry.suffix is None: | |||||
raise InstallationError( | |||||
"Invalid script entry point: %s for req: %s - A callable " | |||||
"suffix is required. Cf https://packaging.python.org/en/" | |||||
"latest/distributing.html#console-scripts for more " | |||||
"information." % (entry, req) | |||||
) | |||||
return maker.script_template % { | |||||
"module": entry.prefix, | |||||
"import_name": entry.suffix.split(".")[0], | |||||
"func": entry.suffix, | |||||
} | |||||
maker._get_script_text = _get_script_text | |||||
maker.script_template = r"""# -*- coding: utf-8 -*- | |||||
import re | |||||
import sys | |||||
from %(module)s import %(import_name)s | |||||
if __name__ == '__main__': | |||||
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) | |||||
sys.exit(%(func)s()) | |||||
""" | |||||
# Special case pip and setuptools to generate versioned wrappers | |||||
# | |||||
# The issue is that some projects (specifically, pip and setuptools) use | |||||
# code in setup.py to create "versioned" entry points - pip2.7 on Python | |||||
# 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into | |||||
# the wheel metadata at build time, and so if the wheel is installed with | |||||
# a *different* version of Python the entry points will be wrong. The | |||||
# correct fix for this is to enhance the metadata to be able to describe | |||||
# such versioned entry points, but that won't happen till Metadata 2.0 is | |||||
# available. | |||||
# In the meantime, projects using versioned entry points will either have | |||||
# incorrect versioned entry points, or they will not be able to distribute | |||||
# "universal" wheels (i.e., they will need a wheel per Python version). | |||||
# | |||||
# Because setuptools and pip are bundled with _ensurepip and virtualenv, | |||||
# we need to use universal wheels. So, as a stopgap until Metadata 2.0, we | |||||
# override the versioned entry points in the wheel and generate the | |||||
# correct ones. This code is purely a short-term measure until Metadata 2.0 | |||||
# is available. | |||||
# | |||||
# To add the level of hack in this section of code, in order to support | |||||
# ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment | |||||
# variable which will control which version scripts get installed. | |||||
# | |||||
# ENSUREPIP_OPTIONS=altinstall | |||||
# - Only pipX.Y and easy_install-X.Y will be generated and installed | |||||
# ENSUREPIP_OPTIONS=install | |||||
# - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note | |||||
# that this option is technically if ENSUREPIP_OPTIONS is set and is | |||||
# not altinstall | |||||
# DEFAULT | |||||
# - The default behavior is to install pip, pipX, pipX.Y, easy_install | |||||
# and easy_install-X.Y. | |||||
pip_script = console.pop('pip', None) | |||||
if pip_script: | |||||
if "ENSUREPIP_OPTIONS" not in os.environ: | |||||
spec = 'pip = ' + pip_script | |||||
generated.extend(maker.make(spec)) | |||||
if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": | |||||
spec = 'pip%s = %s' % (sys.version[:1], pip_script) | |||||
generated.extend(maker.make(spec)) | |||||
spec = 'pip%s = %s' % (sys.version[:3], pip_script) | |||||
generated.extend(maker.make(spec)) | |||||
# Delete any other versioned pip entry points | |||||
pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] | |||||
for k in pip_ep: | |||||
del console[k] | |||||
easy_install_script = console.pop('easy_install', None) | |||||
if easy_install_script: | |||||
if "ENSUREPIP_OPTIONS" not in os.environ: | |||||
spec = 'easy_install = ' + easy_install_script | |||||
generated.extend(maker.make(spec)) | |||||
spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script) | |||||
generated.extend(maker.make(spec)) | |||||
# Delete any other versioned easy_install entry points | |||||
easy_install_ep = [ | |||||
k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) | |||||
] | |||||
for k in easy_install_ep: | |||||
del console[k] | |||||
# Generate the console and GUI entry points specified in the wheel | |||||
if len(console) > 0: | |||||
generated_console_scripts = maker.make_multiple( | |||||
['%s = %s' % kv for kv in console.items()] | |||||
) | |||||
generated.extend(generated_console_scripts) | |||||
if warn_script_location: | |||||
msg = message_about_scripts_not_on_PATH(generated_console_scripts) | |||||
if msg is not None: | |||||
logger.warn(msg) | |||||
if len(gui) > 0: | |||||
generated.extend( | |||||
maker.make_multiple( | |||||
['%s = %s' % kv for kv in gui.items()], | |||||
{'gui': True} | |||||
) | |||||
) | |||||
# Record pip as the installer | |||||
installer = os.path.join(info_dir[0], 'INSTALLER') | |||||
temp_installer = os.path.join(info_dir[0], 'INSTALLER.pip') | |||||
with open(temp_installer, 'wb') as installer_file: | |||||
installer_file.write(b'pip\n') | |||||
shutil.move(temp_installer, installer) | |||||
generated.append(installer) | |||||
# Record details of all files installed | |||||
record = os.path.join(info_dir[0], 'RECORD') | |||||
temp_record = os.path.join(info_dir[0], 'RECORD.pip') | |||||
with open_for_csv(record, 'r') as record_in: | |||||
with open_for_csv(temp_record, 'w+') as record_out: | |||||
reader = csv.reader(record_in) | |||||
writer = csv.writer(record_out) | |||||
for row in reader: | |||||
row[0] = installed.pop(row[0], row[0]) | |||||
if row[0] in changed: | |||||
row[1], row[2] = rehash(row[0]) | |||||
writer.writerow(row) | |||||
for f in generated: | |||||
h, l = rehash(f) | |||||
writer.writerow((normpath(f, lib_dir), h, l)) | |||||
for f in installed: | |||||
writer.writerow((installed[f], '', '')) | |||||
shutil.move(temp_record, record) | |||||
def wheel_version(source_dir): | |||||
""" | |||||
Return the Wheel-Version of an extracted wheel, if possible. | |||||
Otherwise, return False if we couldn't parse / extract it. | |||||
""" | |||||
try: | |||||
dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0] | |||||
wheel_data = dist.get_metadata('WHEEL') | |||||
wheel_data = Parser().parsestr(wheel_data) | |||||
version = wheel_data['Wheel-Version'].strip() | |||||
version = tuple(map(int, version.split('.'))) | |||||
return version | |||||
except: | |||||
return False | |||||
def check_compatibility(version, name): | |||||
""" | |||||
Raises errors or warns if called with an incompatible Wheel-Version. | |||||
Pip should refuse to install a Wheel-Version that's a major series | |||||
ahead of what it's compatible with (e.g 2.0 > 1.1); and warn when | |||||
installing a version only minor version ahead (e.g 1.2 > 1.1). | |||||
version: a 2-tuple representing a Wheel-Version (Major, Minor) | |||||
name: name of wheel or package to raise exception about | |||||
:raises UnsupportedWheel: when an incompatible Wheel-Version is given | |||||
""" | |||||
if not version: | |||||
raise UnsupportedWheel( | |||||
"%s is in an unsupported or invalid wheel" % name | |||||
) | |||||
if version[0] > VERSION_COMPATIBLE[0]: | |||||
raise UnsupportedWheel( | |||||
"%s's Wheel-Version (%s) is not compatible with this version " | |||||
"of pip" % (name, '.'.join(map(str, version))) | |||||
) | |||||
elif version > VERSION_COMPATIBLE: | |||||
logger.warning( | |||||
'Installing from a newer Wheel-Version (%s)', | |||||
'.'.join(map(str, version)), | |||||
) | |||||
class Wheel(object): | |||||
"""A wheel file""" | |||||
# TODO: maybe move the install code into this class | |||||
wheel_file_re = re.compile( | |||||
r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?)) | |||||
((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?) | |||||
\.whl|\.dist-info)$""", | |||||
re.VERBOSE | |||||
) | |||||
def __init__(self, filename): | |||||
""" | |||||
:raises InvalidWheelFilename: when the filename is invalid for a wheel | |||||
""" | |||||
wheel_info = self.wheel_file_re.match(filename) | |||||
if not wheel_info: | |||||
raise InvalidWheelFilename( | |||||
"%s is not a valid wheel filename." % filename | |||||
) | |||||
self.filename = filename | |||||
self.name = wheel_info.group('name').replace('_', '-') | |||||
# we'll assume "_" means "-" due to wheel naming scheme | |||||
# (https://github.com/pypa/pip/issues/1150) | |||||
self.version = wheel_info.group('ver').replace('_', '-') | |||||
self.build_tag = wheel_info.group('build') | |||||
self.pyversions = wheel_info.group('pyver').split('.') | |||||
self.abis = wheel_info.group('abi').split('.') | |||||
self.plats = wheel_info.group('plat').split('.') | |||||
# All the tag combinations from this file | |||||
self.file_tags = { | |||||
(x, y, z) for x in self.pyversions | |||||
for y in self.abis for z in self.plats | |||||
} | |||||
def support_index_min(self, tags=None): | |||||
""" | |||||
Return the lowest index that one of the wheel's file_tag combinations | |||||
achieves in the supported_tags list e.g. if there are 8 supported tags, | |||||
and one of the file tags is first in the list, then return 0. Returns | |||||
None is the wheel is not supported. | |||||
""" | |||||
if tags is None: # for mock | |||||
tags = pep425tags.get_supported() | |||||
indexes = [tags.index(c) for c in self.file_tags if c in tags] | |||||
return min(indexes) if indexes else None | |||||
def supported(self, tags=None): | |||||
"""Is this wheel supported on this system?""" | |||||
if tags is None: # for mock | |||||
tags = pep425tags.get_supported() | |||||
return bool(set(tags).intersection(self.file_tags)) | |||||
class WheelBuilder(object): | |||||
"""Build wheels from a RequirementSet.""" | |||||
def __init__(self, finder, preparer, wheel_cache, | |||||
build_options=None, global_options=None, no_clean=False): | |||||
self.finder = finder | |||||
self.preparer = preparer | |||||
self.wheel_cache = wheel_cache | |||||
self._wheel_dir = preparer.wheel_download_dir | |||||
self.build_options = build_options or [] | |||||
self.global_options = global_options or [] | |||||
self.no_clean = no_clean | |||||
def _build_one(self, req, output_dir, python_tag=None): | |||||
"""Build one wheel. | |||||
:return: The filename of the built wheel, or None if the build failed. | |||||
""" | |||||
# Install build deps into temporary directory (PEP 518) | |||||
with req.build_env: | |||||
return self._build_one_inside_env(req, output_dir, | |||||
python_tag=python_tag) | |||||
def _build_one_inside_env(self, req, output_dir, python_tag=None): | |||||
with TempDirectory(kind="wheel") as temp_dir: | |||||
if self.__build_one(req, temp_dir.path, python_tag=python_tag): | |||||
try: | |||||
wheel_name = os.listdir(temp_dir.path)[0] | |||||
wheel_path = os.path.join(output_dir, wheel_name) | |||||
shutil.move( | |||||
os.path.join(temp_dir.path, wheel_name), wheel_path | |||||
) | |||||
logger.info('Stored in directory: %s', output_dir) | |||||
return wheel_path | |||||
except: | |||||
pass | |||||
# Ignore return, we can't do anything else useful. | |||||
self._clean_one(req) | |||||
return None | |||||
def _base_setup_args(self, req): | |||||
# NOTE: Eventually, we'd want to also -S to the flags here, when we're | |||||
# isolating. Currently, it breaks Python in virtualenvs, because it | |||||
# relies on site.py to find parts of the standard library outside the | |||||
# virtualenv. | |||||
return [ | |||||
sys.executable, '-u', '-c', | |||||
SETUPTOOLS_SHIM % req.setup_py | |||||
] + list(self.global_options) | |||||
def __build_one(self, req, tempd, python_tag=None): | |||||
base_args = self._base_setup_args(req) | |||||
spin_message = 'Running setup.py bdist_wheel for %s' % (req.name,) | |||||
with open_spinner(spin_message) as spinner: | |||||
logger.debug('Destination directory: %s', tempd) | |||||
wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ | |||||
+ self.build_options | |||||
if python_tag is not None: | |||||
wheel_args += ["--python-tag", python_tag] | |||||
try: | |||||
call_subprocess(wheel_args, cwd=req.setup_py_dir, | |||||
show_stdout=False, spinner=spinner) | |||||
return True | |||||
except: | |||||
spinner.finish("error") | |||||
logger.error('Failed building wheel for %s', req.name) | |||||
return False | |||||
def _clean_one(self, req): | |||||
base_args = self._base_setup_args(req) | |||||
logger.info('Running setup.py clean for %s', req.name) | |||||
clean_args = base_args + ['clean', '--all'] | |||||
try: | |||||
call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False) | |||||
return True | |||||
except: | |||||
logger.error('Failed cleaning build dir for %s', req.name) | |||||
return False | |||||
def build(self, requirements, session, autobuilding=False): | |||||
"""Build wheels. | |||||
:param unpack: If True, replace the sdist we built from with the | |||||
newly built wheel, in preparation for installation. | |||||
:return: True if all the wheels built correctly. | |||||
""" | |||||
from pip._internal import index | |||||
building_is_possible = self._wheel_dir or ( | |||||
autobuilding and self.wheel_cache.cache_dir | |||||
) | |||||
assert building_is_possible | |||||
buildset = [] | |||||
for req in requirements: | |||||
if req.constraint: | |||||
continue | |||||
if req.is_wheel: | |||||
if not autobuilding: | |||||
logger.info( | |||||
'Skipping %s, due to already being wheel.', req.name, | |||||
) | |||||
elif autobuilding and req.editable: | |||||
pass | |||||
elif autobuilding and not req.source_dir: | |||||
pass | |||||
elif autobuilding and req.link and not req.link.is_artifact: | |||||
# VCS checkout. Build wheel just for this run. | |||||
buildset.append((req, True)) | |||||
else: | |||||
ephem_cache = False | |||||
if autobuilding: | |||||
link = req.link | |||||
base, ext = link.splitext() | |||||
if index.egg_info_matches(base, None, link) is None: | |||||
# E.g. local directory. Build wheel just for this run. | |||||
ephem_cache = True | |||||
if "binary" not in index.fmt_ctl_formats( | |||||
self.finder.format_control, | |||||
canonicalize_name(req.name)): | |||||
logger.info( | |||||
"Skipping bdist_wheel for %s, due to binaries " | |||||
"being disabled for it.", req.name, | |||||
) | |||||
continue | |||||
buildset.append((req, ephem_cache)) | |||||
if not buildset: | |||||
return True | |||||
# Build the wheels. | |||||
logger.info( | |||||
'Building wheels for collected packages: %s', | |||||
', '.join([req.name for (req, _) in buildset]), | |||||
) | |||||
_cache = self.wheel_cache # shorter name | |||||
with indent_log(): | |||||
build_success, build_failure = [], [] | |||||
for req, ephem in buildset: | |||||
python_tag = None | |||||
if autobuilding: | |||||
python_tag = pep425tags.implementation_tag | |||||
if ephem: | |||||
output_dir = _cache.get_ephem_path_for_link(req.link) | |||||
else: | |||||
output_dir = _cache.get_path_for_link(req.link) | |||||
try: | |||||
ensure_dir(output_dir) | |||||
except OSError as e: | |||||
logger.warning("Building wheel for %s failed: %s", | |||||
req.name, e) | |||||
build_failure.append(req) | |||||
continue | |||||
else: | |||||
output_dir = self._wheel_dir | |||||
wheel_file = self._build_one( | |||||
req, output_dir, | |||||
python_tag=python_tag, | |||||
) | |||||
if wheel_file: | |||||
build_success.append(req) | |||||
if autobuilding: | |||||
# XXX: This is mildly duplicative with prepare_files, | |||||
# but not close enough to pull out to a single common | |||||
# method. | |||||
# The code below assumes temporary source dirs - | |||||
# prevent it doing bad things. | |||||
if req.source_dir and not os.path.exists(os.path.join( | |||||
req.source_dir, PIP_DELETE_MARKER_FILENAME)): | |||||
raise AssertionError( | |||||
"bad source dir - missing marker") | |||||
# Delete the source we built the wheel from | |||||
req.remove_temporary_source() | |||||
# set the build directory again - name is known from | |||||
# the work prepare_files did. | |||||
req.source_dir = req.build_location( | |||||
self.preparer.build_dir | |||||
) | |||||
# Update the link for this. | |||||
req.link = index.Link(path_to_url(wheel_file)) | |||||
assert req.link.is_wheel | |||||
# extract the wheel into the dir | |||||
unpack_url( | |||||
req.link, req.source_dir, None, False, | |||||
session=session, | |||||
) | |||||
else: | |||||
build_failure.append(req) | |||||
# notify success/failure | |||||
if build_success: | |||||
logger.info( | |||||
'Successfully built %s', | |||||
' '.join([req.name for req in build_success]), | |||||
) | |||||
if build_failure: | |||||
logger.info( | |||||
'Failed to build %s', | |||||
' '.join([req.name for req in build_failure]), | |||||
) | |||||
# Return True if all builds were successful | |||||
return len(build_failure) == 0 |
""" | |||||
pip._vendor is for vendoring dependencies of pip to prevent needing pip to | |||||
depend on something external. | |||||
Files inside of pip._vendor should be considered immutable and should only be | |||||
updated to versions from upstream. | |||||
""" | |||||
from __future__ import absolute_import | |||||
import glob | |||||
import os.path | |||||
import sys | |||||
# Downstream redistributors which have debundled our dependencies should also | |||||
# patch this value to be true. This will trigger the additional patching | |||||
# to cause things like "six" to be available as pip. | |||||
DEBUNDLED = False | |||||
# By default, look in this directory for a bunch of .whl files which we will | |||||
# add to the beginning of sys.path before attempting to import anything. This | |||||
# is done to support downstream re-distributors like Debian and Fedora who | |||||
# wish to create their own Wheels for our dependencies to aid in debundling. | |||||
WHEEL_DIR = os.path.abspath(os.path.dirname(__file__)) | |||||
# Define a small helper function to alias our vendored modules to the real ones | |||||
# if the vendored ones do not exist. This idea of this was taken from | |||||
# https://github.com/kennethreitz/requests/pull/2567. | |||||
def vendored(modulename): | |||||
vendored_name = "{0}.{1}".format(__name__, modulename) | |||||
try: | |||||
__import__(vendored_name, globals(), locals(), level=0) | |||||
except ImportError: | |||||
try: | |||||
__import__(modulename, globals(), locals(), level=0) | |||||
except ImportError: | |||||
# We can just silently allow import failures to pass here. If we | |||||
# got to this point it means that ``import pip._vendor.whatever`` | |||||
# failed and so did ``import whatever``. Since we're importing this | |||||
# upfront in an attempt to alias imports, not erroring here will | |||||
# just mean we get a regular import error whenever pip *actually* | |||||
# tries to import one of these modules to use it, which actually | |||||
# gives us a better error message than we would have otherwise | |||||
# gotten. | |||||
pass | |||||
else: | |||||
sys.modules[vendored_name] = sys.modules[modulename] | |||||
base, head = vendored_name.rsplit(".", 1) | |||||
setattr(sys.modules[base], head, sys.modules[modulename]) | |||||
# If we're operating in a debundled setup, then we want to go ahead and trigger | |||||
# the aliasing of our vendored libraries as well as looking for wheels to add | |||||
# to our sys.path. This will cause all of this code to be a no-op typically | |||||
# however downstream redistributors can enable it in a consistent way across | |||||
# all platforms. | |||||
if DEBUNDLED: | |||||
# Actually look inside of WHEEL_DIR to find .whl files and add them to the | |||||
# front of our sys.path. | |||||
sys.path[:] = glob.glob(os.path.join(WHEEL_DIR, "*.whl")) + sys.path | |||||
# Actually alias all of our vendored dependencies. | |||||
vendored("cachecontrol") | |||||
vendored("colorama") | |||||
vendored("distlib") | |||||
vendored("distro") | |||||
vendored("html5lib") | |||||
vendored("lockfile") | |||||
vendored("six") | |||||
vendored("six.moves") | |||||
vendored("six.moves.urllib") | |||||
vendored("six.moves.urllib.parse") | |||||
vendored("packaging") | |||||
vendored("packaging.version") | |||||
vendored("packaging.specifiers") | |||||
vendored("pkg_resources") | |||||
vendored("progress") | |||||
vendored("pytoml") | |||||
vendored("retrying") | |||||
vendored("requests") | |||||
vendored("requests.packages") | |||||
vendored("requests.packages.urllib3") | |||||
vendored("requests.packages.urllib3._collections") | |||||
vendored("requests.packages.urllib3.connection") | |||||
vendored("requests.packages.urllib3.connectionpool") | |||||
vendored("requests.packages.urllib3.contrib") | |||||
vendored("requests.packages.urllib3.contrib.ntlmpool") | |||||
vendored("requests.packages.urllib3.contrib.pyopenssl") | |||||
vendored("requests.packages.urllib3.exceptions") | |||||
vendored("requests.packages.urllib3.fields") | |||||
vendored("requests.packages.urllib3.filepost") | |||||
vendored("requests.packages.urllib3.packages") | |||||
vendored("requests.packages.urllib3.packages.ordered_dict") | |||||
vendored("requests.packages.urllib3.packages.six") | |||||
vendored("requests.packages.urllib3.packages.ssl_match_hostname") | |||||
vendored("requests.packages.urllib3.packages.ssl_match_hostname." | |||||
"_implementation") | |||||
vendored("requests.packages.urllib3.poolmanager") | |||||
vendored("requests.packages.urllib3.request") | |||||
vendored("requests.packages.urllib3.response") | |||||
vendored("requests.packages.urllib3.util") | |||||
vendored("requests.packages.urllib3.util.connection") | |||||
vendored("requests.packages.urllib3.util.request") | |||||
vendored("requests.packages.urllib3.util.response") | |||||
vendored("requests.packages.urllib3.util.retry") | |||||
vendored("requests.packages.urllib3.util.ssl_") | |||||
vendored("requests.packages.urllib3.util.timeout") | |||||
vendored("requests.packages.urllib3.util.url") |
#!/usr/bin/env python | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2005-2010 ActiveState Software Inc. | |||||
# Copyright (c) 2013 Eddy Petrișor | |||||
"""Utilities for determining application-specific dirs. | |||||
See <http://github.com/ActiveState/appdirs> for details and usage. | |||||
""" | |||||
# Dev Notes: | |||||
# - MSDN on where to store app data files: | |||||
# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 | |||||
# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html | |||||
# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html | |||||
__version_info__ = (1, 4, 3) | |||||
__version__ = '.'.join(map(str, __version_info__)) | |||||
import sys | |||||
import os | |||||
PY3 = sys.version_info[0] == 3 | |||||
if PY3: | |||||
unicode = str | |||||
if sys.platform.startswith('java'): | |||||
import platform | |||||
os_name = platform.java_ver()[3][0] | |||||
if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc. | |||||
system = 'win32' | |||||
elif os_name.startswith('Mac'): # "Mac OS X", etc. | |||||
system = 'darwin' | |||||
else: # "Linux", "SunOS", "FreeBSD", etc. | |||||
# Setting this to "linux2" is not ideal, but only Windows or Mac | |||||
# are actually checked for and the rest of the module expects | |||||
# *sys.platform* style strings. | |||||
system = 'linux2' | |||||
else: | |||||
system = sys.platform | |||||
def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): | |||||
r"""Return full path to the user-specific data dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"roaming" (boolean, default False) can be set True to use the Windows | |||||
roaming appdata directory. That means that for users on a Windows | |||||
network setup for roaming profiles, this user data will be | |||||
sync'd on login. See | |||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> | |||||
for a discussion of issues. | |||||
Typical user data directories are: | |||||
Mac OS X: ~/Library/Application Support/<AppName> | |||||
Unix: ~/.local/share/<AppName> # or in $XDG_DATA_HOME, if defined | |||||
Win XP (not roaming): C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName> | |||||
Win XP (roaming): C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName> | |||||
Win 7 (not roaming): C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName> | |||||
Win 7 (roaming): C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName> | |||||
For Unix, we follow the XDG spec and support $XDG_DATA_HOME. | |||||
That means, by default "~/.local/share/<AppName>". | |||||
""" | |||||
if system == "win32": | |||||
if appauthor is None: | |||||
appauthor = appname | |||||
const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" | |||||
path = os.path.normpath(_get_win_folder(const)) | |||||
if appname: | |||||
if appauthor is not False: | |||||
path = os.path.join(path, appauthor, appname) | |||||
else: | |||||
path = os.path.join(path, appname) | |||||
elif system == 'darwin': | |||||
path = os.path.expanduser('~/Library/Application Support/') | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
else: | |||||
path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
return path | |||||
def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): | |||||
r"""Return full path to the user-shared data dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"multipath" is an optional parameter only applicable to *nix | |||||
which indicates that the entire list of data dirs should be | |||||
returned. By default, the first item from XDG_DATA_DIRS is | |||||
returned, or '/usr/local/share/<AppName>', | |||||
if XDG_DATA_DIRS is not set | |||||
Typical site data directories are: | |||||
Mac OS X: /Library/Application Support/<AppName> | |||||
Unix: /usr/local/share/<AppName> or /usr/share/<AppName> | |||||
Win XP: C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName> | |||||
Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) | |||||
Win 7: C:\ProgramData\<AppAuthor>\<AppName> # Hidden, but writeable on Win 7. | |||||
For Unix, this is using the $XDG_DATA_DIRS[0] default. | |||||
WARNING: Do not use this on Windows. See the Vista-Fail note above for why. | |||||
""" | |||||
if system == "win32": | |||||
if appauthor is None: | |||||
appauthor = appname | |||||
path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) | |||||
if appname: | |||||
if appauthor is not False: | |||||
path = os.path.join(path, appauthor, appname) | |||||
else: | |||||
path = os.path.join(path, appname) | |||||
elif system == 'darwin': | |||||
path = os.path.expanduser('/Library/Application Support') | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
else: | |||||
# XDG default for $XDG_DATA_DIRS | |||||
# only first, if multipath is False | |||||
path = os.getenv('XDG_DATA_DIRS', | |||||
os.pathsep.join(['/usr/local/share', '/usr/share'])) | |||||
pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] | |||||
if appname: | |||||
if version: | |||||
appname = os.path.join(appname, version) | |||||
pathlist = [os.sep.join([x, appname]) for x in pathlist] | |||||
if multipath: | |||||
path = os.pathsep.join(pathlist) | |||||
else: | |||||
path = pathlist[0] | |||||
return path | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
return path | |||||
def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): | |||||
r"""Return full path to the user-specific config dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"roaming" (boolean, default False) can be set True to use the Windows | |||||
roaming appdata directory. That means that for users on a Windows | |||||
network setup for roaming profiles, this user data will be | |||||
sync'd on login. See | |||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> | |||||
for a discussion of issues. | |||||
Typical user config directories are: | |||||
Mac OS X: same as user_data_dir | |||||
Unix: ~/.config/<AppName> # or in $XDG_CONFIG_HOME, if defined | |||||
Win *: same as user_data_dir | |||||
For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. | |||||
That means, by default "~/.config/<AppName>". | |||||
""" | |||||
if system in ["win32", "darwin"]: | |||||
path = user_data_dir(appname, appauthor, None, roaming) | |||||
else: | |||||
path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
return path | |||||
def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): | |||||
r"""Return full path to the user-shared data dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"multipath" is an optional parameter only applicable to *nix | |||||
which indicates that the entire list of config dirs should be | |||||
returned. By default, the first item from XDG_CONFIG_DIRS is | |||||
returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set | |||||
Typical site config directories are: | |||||
Mac OS X: same as site_data_dir | |||||
Unix: /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in | |||||
$XDG_CONFIG_DIRS | |||||
Win *: same as site_data_dir | |||||
Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) | |||||
For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False | |||||
WARNING: Do not use this on Windows. See the Vista-Fail note above for why. | |||||
""" | |||||
if system in ["win32", "darwin"]: | |||||
path = site_data_dir(appname, appauthor) | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
else: | |||||
# XDG default for $XDG_CONFIG_DIRS | |||||
# only first, if multipath is False | |||||
path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') | |||||
pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] | |||||
if appname: | |||||
if version: | |||||
appname = os.path.join(appname, version) | |||||
pathlist = [os.sep.join([x, appname]) for x in pathlist] | |||||
if multipath: | |||||
path = os.pathsep.join(pathlist) | |||||
else: | |||||
path = pathlist[0] | |||||
return path | |||||
def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): | |||||
r"""Return full path to the user-specific cache dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"opinion" (boolean) can be False to disable the appending of | |||||
"Cache" to the base app data dir for Windows. See | |||||
discussion below. | |||||
Typical user cache directories are: | |||||
Mac OS X: ~/Library/Caches/<AppName> | |||||
Unix: ~/.cache/<AppName> (XDG default) | |||||
Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache | |||||
Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache | |||||
On Windows the only suggestion in the MSDN docs is that local settings go in | |||||
the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming | |||||
app data dir (the default returned by `user_data_dir` above). Apps typically | |||||
put cache data somewhere *under* the given dir here. Some examples: | |||||
...\Mozilla\Firefox\Profiles\<ProfileName>\Cache | |||||
...\Acme\SuperApp\Cache\1.0 | |||||
OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. | |||||
This can be disabled with the `opinion=False` option. | |||||
""" | |||||
if system == "win32": | |||||
if appauthor is None: | |||||
appauthor = appname | |||||
path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) | |||||
if appname: | |||||
if appauthor is not False: | |||||
path = os.path.join(path, appauthor, appname) | |||||
else: | |||||
path = os.path.join(path, appname) | |||||
if opinion: | |||||
path = os.path.join(path, "Cache") | |||||
elif system == 'darwin': | |||||
path = os.path.expanduser('~/Library/Caches') | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
else: | |||||
path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
return path | |||||
def user_state_dir(appname=None, appauthor=None, version=None, roaming=False): | |||||
r"""Return full path to the user-specific state dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"roaming" (boolean, default False) can be set True to use the Windows | |||||
roaming appdata directory. That means that for users on a Windows | |||||
network setup for roaming profiles, this user data will be | |||||
sync'd on login. See | |||||
<http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> | |||||
for a discussion of issues. | |||||
Typical user state directories are: | |||||
Mac OS X: same as user_data_dir | |||||
Unix: ~/.local/state/<AppName> # or in $XDG_STATE_HOME, if defined | |||||
Win *: same as user_data_dir | |||||
For Unix, we follow this Debian proposal <https://wiki.debian.org/XDGBaseDirectorySpecification#state> | |||||
to extend the XDG spec and support $XDG_STATE_HOME. | |||||
That means, by default "~/.local/state/<AppName>". | |||||
""" | |||||
if system in ["win32", "darwin"]: | |||||
path = user_data_dir(appname, appauthor, None, roaming) | |||||
else: | |||||
path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state")) | |||||
if appname: | |||||
path = os.path.join(path, appname) | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
return path | |||||
def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): | |||||
r"""Return full path to the user-specific log dir for this application. | |||||
"appname" is the name of application. | |||||
If None, just the system directory is returned. | |||||
"appauthor" (only used on Windows) is the name of the | |||||
appauthor or distributing body for this application. Typically | |||||
it is the owning company name. This falls back to appname. You may | |||||
pass False to disable it. | |||||
"version" is an optional version path element to append to the | |||||
path. You might want to use this if you want multiple versions | |||||
of your app to be able to run independently. If used, this | |||||
would typically be "<major>.<minor>". | |||||
Only applied when appname is present. | |||||
"opinion" (boolean) can be False to disable the appending of | |||||
"Logs" to the base app data dir for Windows, and "log" to the | |||||
base cache dir for Unix. See discussion below. | |||||
Typical user log directories are: | |||||
Mac OS X: ~/Library/Logs/<AppName> | |||||
Unix: ~/.cache/<AppName>/log # or under $XDG_CACHE_HOME if defined | |||||
Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs | |||||
Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs | |||||
On Windows the only suggestion in the MSDN docs is that local settings | |||||
go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in | |||||
examples of what some windows apps use for a logs dir.) | |||||
OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` | |||||
value for Windows and appends "log" to the user cache dir for Unix. | |||||
This can be disabled with the `opinion=False` option. | |||||
""" | |||||
if system == "darwin": | |||||
path = os.path.join( | |||||
os.path.expanduser('~/Library/Logs'), | |||||
appname) | |||||
elif system == "win32": | |||||
path = user_data_dir(appname, appauthor, version) | |||||
version = False | |||||
if opinion: | |||||
path = os.path.join(path, "Logs") | |||||
else: | |||||
path = user_cache_dir(appname, appauthor, version) | |||||
version = False | |||||
if opinion: | |||||
path = os.path.join(path, "log") | |||||
if appname and version: | |||||
path = os.path.join(path, version) | |||||
return path | |||||
class AppDirs(object): | |||||
"""Convenience wrapper for getting application dirs.""" | |||||
def __init__(self, appname=None, appauthor=None, version=None, | |||||
roaming=False, multipath=False): | |||||
self.appname = appname | |||||
self.appauthor = appauthor | |||||
self.version = version | |||||
self.roaming = roaming | |||||
self.multipath = multipath | |||||
@property | |||||
def user_data_dir(self): | |||||
return user_data_dir(self.appname, self.appauthor, | |||||
version=self.version, roaming=self.roaming) | |||||
@property | |||||
def site_data_dir(self): | |||||
return site_data_dir(self.appname, self.appauthor, | |||||
version=self.version, multipath=self.multipath) | |||||
@property | |||||
def user_config_dir(self): | |||||
return user_config_dir(self.appname, self.appauthor, | |||||
version=self.version, roaming=self.roaming) | |||||
@property | |||||
def site_config_dir(self): | |||||
return site_config_dir(self.appname, self.appauthor, | |||||
version=self.version, multipath=self.multipath) | |||||
@property | |||||
def user_cache_dir(self): | |||||
return user_cache_dir(self.appname, self.appauthor, | |||||
version=self.version) | |||||
@property | |||||
def user_state_dir(self): | |||||
return user_state_dir(self.appname, self.appauthor, | |||||
version=self.version) | |||||
@property | |||||
def user_log_dir(self): | |||||
return user_log_dir(self.appname, self.appauthor, | |||||
version=self.version) | |||||
#---- internal support stuff | |||||
def _get_win_folder_from_registry(csidl_name): | |||||
"""This is a fallback technique at best. I'm not sure if using the | |||||
registry for this guarantees us the correct answer for all CSIDL_* | |||||
names. | |||||
""" | |||||
if PY3: | |||||
import winreg as _winreg | |||||
else: | |||||
import _winreg | |||||
shell_folder_name = { | |||||
"CSIDL_APPDATA": "AppData", | |||||
"CSIDL_COMMON_APPDATA": "Common AppData", | |||||
"CSIDL_LOCAL_APPDATA": "Local AppData", | |||||
}[csidl_name] | |||||
key = _winreg.OpenKey( | |||||
_winreg.HKEY_CURRENT_USER, | |||||
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" | |||||
) | |||||
dir, type = _winreg.QueryValueEx(key, shell_folder_name) | |||||
return dir | |||||
def _get_win_folder_with_pywin32(csidl_name): | |||||
from win32com.shell import shellcon, shell | |||||
dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) | |||||
# Try to make this a unicode path because SHGetFolderPath does | |||||
# not return unicode strings when there is unicode data in the | |||||
# path. | |||||
try: | |||||
dir = unicode(dir) | |||||
# Downgrade to short path name if have highbit chars. See | |||||
# <http://bugs.activestate.com/show_bug.cgi?id=85099>. | |||||
has_high_char = False | |||||
for c in dir: | |||||
if ord(c) > 255: | |||||
has_high_char = True | |||||
break | |||||
if has_high_char: | |||||
try: | |||||
import win32api | |||||
dir = win32api.GetShortPathName(dir) | |||||
except ImportError: | |||||
pass | |||||
except UnicodeError: | |||||
pass | |||||
return dir | |||||
def _get_win_folder_with_ctypes(csidl_name): | |||||
import ctypes | |||||
csidl_const = { | |||||
"CSIDL_APPDATA": 26, | |||||
"CSIDL_COMMON_APPDATA": 35, | |||||
"CSIDL_LOCAL_APPDATA": 28, | |||||
}[csidl_name] | |||||
buf = ctypes.create_unicode_buffer(1024) | |||||
ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) | |||||
# Downgrade to short path name if have highbit chars. See | |||||
# <http://bugs.activestate.com/show_bug.cgi?id=85099>. | |||||
has_high_char = False | |||||
for c in buf: | |||||
if ord(c) > 255: | |||||
has_high_char = True | |||||
break | |||||
if has_high_char: | |||||
buf2 = ctypes.create_unicode_buffer(1024) | |||||
if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): | |||||
buf = buf2 | |||||
return buf.value | |||||
def _get_win_folder_with_jna(csidl_name): | |||||
import array | |||||
from com.sun import jna | |||||
from com.sun.jna.platform import win32 | |||||
buf_size = win32.WinDef.MAX_PATH * 2 | |||||
buf = array.zeros('c', buf_size) | |||||
shell = win32.Shell32.INSTANCE | |||||
shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf) | |||||
dir = jna.Native.toString(buf.tostring()).rstrip("\0") | |||||
# Downgrade to short path name if have highbit chars. See | |||||
# <http://bugs.activestate.com/show_bug.cgi?id=85099>. | |||||
has_high_char = False | |||||
for c in dir: | |||||
if ord(c) > 255: | |||||
has_high_char = True | |||||
break | |||||
if has_high_char: | |||||
buf = array.zeros('c', buf_size) | |||||
kernel = win32.Kernel32.INSTANCE | |||||
if kernel.GetShortPathName(dir, buf, buf_size): | |||||
dir = jna.Native.toString(buf.tostring()).rstrip("\0") | |||||
return dir | |||||
if system == "win32": | |||||
try: | |||||
from ctypes import windll | |||||
_get_win_folder = _get_win_folder_with_ctypes | |||||
except ImportError: | |||||
try: | |||||
import com.sun.jna | |||||
_get_win_folder = _get_win_folder_with_jna | |||||
except ImportError: | |||||
_get_win_folder = _get_win_folder_from_registry | |||||
#---- self test code | |||||
if __name__ == "__main__": | |||||
appname = "MyApp" | |||||
appauthor = "MyCompany" | |||||
props = ("user_data_dir", | |||||
"user_config_dir", | |||||
"user_cache_dir", | |||||
"user_state_dir", | |||||
"user_log_dir", | |||||
"site_data_dir", | |||||
"site_config_dir") | |||||
print("-- app dirs %s --" % __version__) | |||||
print("-- app dirs (with optional 'version')") | |||||
dirs = AppDirs(appname, appauthor, version="1.0") | |||||
for prop in props: | |||||
print("%s: %s" % (prop, getattr(dirs, prop))) | |||||
print("\n-- app dirs (without optional 'version')") | |||||
dirs = AppDirs(appname, appauthor) | |||||
for prop in props: | |||||
print("%s: %s" % (prop, getattr(dirs, prop))) | |||||
print("\n-- app dirs (without optional 'appauthor')") | |||||
dirs = AppDirs(appname) | |||||
for prop in props: | |||||
print("%s: %s" % (prop, getattr(dirs, prop))) | |||||
print("\n-- app dirs (with disabled 'appauthor')") | |||||
dirs = AppDirs(appname, appauthor=False) | |||||
for prop in props: | |||||
print("%s: %s" % (prop, getattr(dirs, prop))) |
"""CacheControl import Interface. | |||||
Make it easy to import from cachecontrol without long namespaces. | |||||
""" | |||||
__author__ = 'Eric Larson' | |||||
__email__ = 'eric@ionrock.org' | |||||
__version__ = '0.12.4' | |||||
from .wrapper import CacheControl | |||||
from .adapter import CacheControlAdapter | |||||
from .controller import CacheController |
import logging | |||||
from pip._vendor import requests | |||||
from pip._vendor.cachecontrol.adapter import CacheControlAdapter | |||||
from pip._vendor.cachecontrol.cache import DictCache | |||||
from pip._vendor.cachecontrol.controller import logger | |||||
from argparse import ArgumentParser | |||||
def setup_logging(): | |||||
logger.setLevel(logging.DEBUG) | |||||
handler = logging.StreamHandler() | |||||
logger.addHandler(handler) | |||||
def get_session(): | |||||
adapter = CacheControlAdapter( | |||||
DictCache(), | |||||
cache_etags=True, | |||||
serializer=None, | |||||
heuristic=None, | |||||
) | |||||
sess = requests.Session() | |||||
sess.mount('http://', adapter) | |||||
sess.mount('https://', adapter) | |||||
sess.cache_controller = adapter.controller | |||||
return sess | |||||
def get_args(): | |||||
parser = ArgumentParser() | |||||
parser.add_argument('url', help='The URL to try and cache') | |||||
return parser.parse_args() | |||||
def main(args=None): | |||||
args = get_args() | |||||
sess = get_session() | |||||
# Make a request to get a response | |||||
resp = sess.get(args.url) | |||||
# Turn on logging | |||||
setup_logging() | |||||
# try setting the cache | |||||
sess.cache_controller.cache_response(resp.request, resp.raw) | |||||
# Now try to get it | |||||
if sess.cache_controller.cached_request(resp.request): | |||||
print('Cached!') | |||||
else: | |||||
print('Not cached :(') | |||||
if __name__ == '__main__': | |||||
main() |
import types | |||||
import functools | |||||
import zlib | |||||
from pip._vendor.requests.adapters import HTTPAdapter | |||||
from .controller import CacheController | |||||
from .cache import DictCache | |||||
from .filewrapper import CallbackFileWrapper | |||||
class CacheControlAdapter(HTTPAdapter): | |||||
invalidating_methods = set(['PUT', 'DELETE']) | |||||
def __init__(self, cache=None, | |||||
cache_etags=True, | |||||
controller_class=None, | |||||
serializer=None, | |||||
heuristic=None, | |||||
cacheable_methods=None, | |||||
*args, **kw): | |||||
super(CacheControlAdapter, self).__init__(*args, **kw) | |||||
self.cache = cache or DictCache() | |||||
self.heuristic = heuristic | |||||
self.cacheable_methods = cacheable_methods or ('GET',) | |||||
controller_factory = controller_class or CacheController | |||||
self.controller = controller_factory( | |||||
self.cache, | |||||
cache_etags=cache_etags, | |||||
serializer=serializer, | |||||
) | |||||
def send(self, request, cacheable_methods=None, **kw): | |||||
""" | |||||
Send a request. Use the request information to see if it | |||||
exists in the cache and cache the response if we need to and can. | |||||
""" | |||||
cacheable = cacheable_methods or self.cacheable_methods | |||||
if request.method in cacheable: | |||||
try: | |||||
cached_response = self.controller.cached_request(request) | |||||
except zlib.error: | |||||
cached_response = None | |||||
if cached_response: | |||||
return self.build_response(request, cached_response, | |||||
from_cache=True) | |||||
# check for etags and add headers if appropriate | |||||
request.headers.update( | |||||
self.controller.conditional_headers(request) | |||||
) | |||||
resp = super(CacheControlAdapter, self).send(request, **kw) | |||||
return resp | |||||
def build_response(self, request, response, from_cache=False, | |||||
cacheable_methods=None): | |||||
""" | |||||
Build a response by making a request or using the cache. | |||||
This will end up calling send and returning a potentially | |||||
cached response | |||||
""" | |||||
cacheable = cacheable_methods or self.cacheable_methods | |||||
if not from_cache and request.method in cacheable: | |||||
# Check for any heuristics that might update headers | |||||
# before trying to cache. | |||||
if self.heuristic: | |||||
response = self.heuristic.apply(response) | |||||
# apply any expiration heuristics | |||||
if response.status == 304: | |||||
# We must have sent an ETag request. This could mean | |||||
# that we've been expired already or that we simply | |||||
# have an etag. In either case, we want to try and | |||||
# update the cache if that is the case. | |||||
cached_response = self.controller.update_cached_response( | |||||
request, response | |||||
) | |||||
if cached_response is not response: | |||||
from_cache = True | |||||
# We are done with the server response, read a | |||||
# possible response body (compliant servers will | |||||
# not return one, but we cannot be 100% sure) and | |||||
# release the connection back to the pool. | |||||
response.read(decode_content=False) | |||||
response.release_conn() | |||||
response = cached_response | |||||
# We always cache the 301 responses | |||||
elif response.status == 301: | |||||
self.controller.cache_response(request, response) | |||||
else: | |||||
# Wrap the response file with a wrapper that will cache the | |||||
# response when the stream has been consumed. | |||||
response._fp = CallbackFileWrapper( | |||||
response._fp, | |||||
functools.partial( | |||||
self.controller.cache_response, | |||||
request, | |||||
response, | |||||
) | |||||
) | |||||
if response.chunked: | |||||
super_update_chunk_length = response._update_chunk_length | |||||
def _update_chunk_length(self): | |||||
super_update_chunk_length() | |||||
if self.chunk_left == 0: | |||||
self._fp._close() | |||||
response._update_chunk_length = types.MethodType(_update_chunk_length, response) | |||||
resp = super(CacheControlAdapter, self).build_response( | |||||
request, response | |||||
) | |||||
# See if we should invalidate the cache. | |||||
if request.method in self.invalidating_methods and resp.ok: | |||||
cache_url = self.controller.cache_url(request.url) | |||||
self.cache.delete(cache_url) | |||||
# Give the request a from_cache attr to let people use it | |||||
resp.from_cache = from_cache | |||||
return resp | |||||
def close(self): | |||||
self.cache.close() | |||||
super(CacheControlAdapter, self).close() |
""" | |||||
The cache object API for implementing caches. The default is a thread | |||||
safe in-memory dictionary. | |||||
""" | |||||
from threading import Lock | |||||
class BaseCache(object): | |||||
def get(self, key): | |||||
raise NotImplemented() | |||||
def set(self, key, value): | |||||
raise NotImplemented() | |||||
def delete(self, key): | |||||
raise NotImplemented() | |||||
def close(self): | |||||
pass | |||||
class DictCache(BaseCache): | |||||
def __init__(self, init_dict=None): | |||||
self.lock = Lock() | |||||
self.data = init_dict or {} | |||||
def get(self, key): | |||||
return self.data.get(key, None) | |||||
def set(self, key, value): | |||||
with self.lock: | |||||
self.data.update({key: value}) | |||||
def delete(self, key): | |||||
with self.lock: | |||||
if key in self.data: | |||||
self.data.pop(key) |
from .file_cache import FileCache # noqa | |||||
from .redis_cache import RedisCache # noqa |
import hashlib | |||||
import os | |||||
from textwrap import dedent | |||||
from ..cache import BaseCache | |||||
from ..controller import CacheController | |||||
try: | |||||
FileNotFoundError | |||||
except NameError: | |||||
# py2.X | |||||
FileNotFoundError = OSError | |||||
def _secure_open_write(filename, fmode): | |||||
# We only want to write to this file, so open it in write only mode | |||||
flags = os.O_WRONLY | |||||
# os.O_CREAT | os.O_EXCL will fail if the file already exists, so we only | |||||
# will open *new* files. | |||||
# We specify this because we want to ensure that the mode we pass is the | |||||
# mode of the file. | |||||
flags |= os.O_CREAT | os.O_EXCL | |||||
# Do not follow symlinks to prevent someone from making a symlink that | |||||
# we follow and insecurely open a cache file. | |||||
if hasattr(os, "O_NOFOLLOW"): | |||||
flags |= os.O_NOFOLLOW | |||||
# On Windows we'll mark this file as binary | |||||
if hasattr(os, "O_BINARY"): | |||||
flags |= os.O_BINARY | |||||
# Before we open our file, we want to delete any existing file that is | |||||
# there | |||||
try: | |||||
os.remove(filename) | |||||
except (IOError, OSError): | |||||
# The file must not exist already, so we can just skip ahead to opening | |||||
pass | |||||
# Open our file, the use of os.O_CREAT | os.O_EXCL will ensure that if a | |||||
# race condition happens between the os.remove and this line, that an | |||||
# error will be raised. Because we utilize a lockfile this should only | |||||
# happen if someone is attempting to attack us. | |||||
fd = os.open(filename, flags, fmode) | |||||
try: | |||||
return os.fdopen(fd, "wb") | |||||
except: | |||||
# An error occurred wrapping our FD in a file object | |||||
os.close(fd) | |||||
raise | |||||
class FileCache(BaseCache): | |||||
def __init__(self, directory, forever=False, filemode=0o0600, | |||||
dirmode=0o0700, use_dir_lock=None, lock_class=None): | |||||
if use_dir_lock is not None and lock_class is not None: | |||||
raise ValueError("Cannot use use_dir_lock and lock_class together") | |||||
try: | |||||
from pip._vendor.lockfile import LockFile | |||||
from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile | |||||
except ImportError: | |||||
notice = dedent(""" | |||||
NOTE: In order to use the FileCache you must have | |||||
lockfile installed. You can install it via pip: | |||||
pip install lockfile | |||||
""") | |||||
raise ImportError(notice) | |||||
else: | |||||
if use_dir_lock: | |||||
lock_class = MkdirLockFile | |||||
elif lock_class is None: | |||||
lock_class = LockFile | |||||
self.directory = directory | |||||
self.forever = forever | |||||
self.filemode = filemode | |||||
self.dirmode = dirmode | |||||
self.lock_class = lock_class | |||||
@staticmethod | |||||
def encode(x): | |||||
return hashlib.sha224(x.encode()).hexdigest() | |||||
def _fn(self, name): | |||||
# NOTE: This method should not change as some may depend on it. | |||||
# See: https://github.com/ionrock/cachecontrol/issues/63 | |||||
hashed = self.encode(name) | |||||
parts = list(hashed[:5]) + [hashed] | |||||
return os.path.join(self.directory, *parts) | |||||
def get(self, key): | |||||
name = self._fn(key) | |||||
if not os.path.exists(name): | |||||
return None | |||||
with open(name, 'rb') as fh: | |||||
return fh.read() | |||||
def set(self, key, value): | |||||
name = self._fn(key) | |||||
# Make sure the directory exists | |||||
try: | |||||
os.makedirs(os.path.dirname(name), self.dirmode) | |||||
except (IOError, OSError): | |||||
pass | |||||
with self.lock_class(name) as lock: | |||||
# Write our actual file | |||||
with _secure_open_write(lock.path, self.filemode) as fh: | |||||
fh.write(value) | |||||
def delete(self, key): | |||||
name = self._fn(key) | |||||
if not self.forever: | |||||
try: | |||||
os.remove(name) | |||||
except FileNotFoundError: | |||||
pass | |||||
def url_to_file_path(url, filecache): | |||||
"""Return the file cache path based on the URL. | |||||
This does not ensure the file exists! | |||||
""" | |||||
key = CacheController.cache_url(url) | |||||
return filecache._fn(key) |
from __future__ import division | |||||
from datetime import datetime | |||||
from pip._vendor.cachecontrol.cache import BaseCache | |||||
def total_seconds(td): | |||||
"""Python 2.6 compatability""" | |||||
if hasattr(td, 'total_seconds'): | |||||
return int(td.total_seconds()) | |||||
ms = td.microseconds | |||||
secs = (td.seconds + td.days * 24 * 3600) | |||||
return int((ms + secs * 10**6) / 10**6) | |||||
class RedisCache(BaseCache): | |||||
def __init__(self, conn): | |||||
self.conn = conn | |||||
def get(self, key): | |||||
return self.conn.get(key) | |||||
def set(self, key, value, expires=None): | |||||
if not expires: | |||||
self.conn.set(key, value) | |||||
else: | |||||
expires = expires - datetime.utcnow() | |||||
self.conn.setex(key, total_seconds(expires), value) | |||||
def delete(self, key): | |||||
self.conn.delete(key) | |||||
def clear(self): | |||||
"""Helper for clearing all the keys in a database. Use with | |||||
caution!""" | |||||
for key in self.conn.keys(): | |||||
self.conn.delete(key) | |||||
def close(self): | |||||
"""Redis uses connection pooling, no need to close the connection.""" | |||||
pass |
try: | |||||
from urllib.parse import urljoin | |||||
except ImportError: | |||||
from urlparse import urljoin | |||||
try: | |||||
import cPickle as pickle | |||||
except ImportError: | |||||
import pickle | |||||
# Handle the case where the requests module has been patched to not have | |||||
# urllib3 bundled as part of its source. | |||||
try: | |||||
from pip._vendor.requests.packages.urllib3.response import HTTPResponse | |||||
except ImportError: | |||||
from pip._vendor.urllib3.response import HTTPResponse | |||||
try: | |||||
from pip._vendor.requests.packages.urllib3.util import is_fp_closed | |||||
except ImportError: | |||||
from pip._vendor.urllib3.util import is_fp_closed | |||||
# Replicate some six behaviour | |||||
try: | |||||
text_type = unicode | |||||
except NameError: | |||||
text_type = str |