You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

configuration.py 13KB

5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. """Configuration management setup
  2. Some terminology:
  3. - name
  4. As written in config files.
  5. - value
  6. Value associated with a name
  7. - key
  8. Name combined with it's section (section.name)
  9. - variant
  10. A single word describing where the configuration key-value pair came from
  11. """
  12. import locale
  13. import logging
  14. import os
  15. from pip._vendor import six
  16. from pip._vendor.six.moves import configparser
  17. from pip._internal.exceptions import (
  18. ConfigurationError, ConfigurationFileCouldNotBeLoaded,
  19. )
  20. from pip._internal.locations import (
  21. legacy_config_file, new_config_file, running_under_virtualenv,
  22. site_config_files, venv_config_file,
  23. )
  24. from pip._internal.utils.misc import ensure_dir, enum
  25. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  26. if MYPY_CHECK_RUNNING:
  27. from typing import ( # noqa: F401
  28. Any, Dict, Iterable, List, NewType, Optional, Tuple
  29. )
  30. RawConfigParser = configparser.RawConfigParser # Shorthand
  31. Kind = NewType("Kind", str)
  32. logger = logging.getLogger(__name__)
  33. # NOTE: Maybe use the optionx attribute to normalize keynames.
  34. def _normalize_name(name):
  35. # type: (str) -> str
  36. """Make a name consistent regardless of source (environment or file)
  37. """
  38. name = name.lower().replace('_', '-')
  39. if name.startswith('--'):
  40. name = name[2:] # only prefer long opts
  41. return name
  42. def _disassemble_key(name):
  43. # type: (str) -> List[str]
  44. return name.split(".", 1)
  45. # The kinds of configurations there are.
  46. kinds = enum(
  47. USER="user", # User Specific
  48. GLOBAL="global", # System Wide
  49. VENV="venv", # Virtual Environment Specific
  50. ENV="env", # from PIP_CONFIG_FILE
  51. ENV_VAR="env-var", # from Environment Variables
  52. )
  53. class Configuration(object):
  54. """Handles management of configuration.
  55. Provides an interface to accessing and managing configuration files.
  56. This class converts provides an API that takes "section.key-name" style
  57. keys and stores the value associated with it as "key-name" under the
  58. section "section".
  59. This allows for a clean interface wherein the both the section and the
  60. key-name are preserved in an easy to manage form in the configuration files
  61. and the data stored is also nice.
  62. """
  63. def __init__(self, isolated, load_only=None):
  64. # type: (bool, Kind) -> None
  65. super(Configuration, self).__init__()
  66. _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.VENV, None]
  67. if load_only not in _valid_load_only:
  68. raise ConfigurationError(
  69. "Got invalid value for load_only - should be one of {}".format(
  70. ", ".join(map(repr, _valid_load_only[:-1]))
  71. )
  72. )
  73. self.isolated = isolated # type: bool
  74. self.load_only = load_only # type: Optional[Kind]
  75. # The order here determines the override order.
  76. self._override_order = [
  77. kinds.GLOBAL, kinds.USER, kinds.VENV, kinds.ENV, kinds.ENV_VAR
  78. ]
  79. self._ignore_env_names = ["version", "help"]
  80. # Because we keep track of where we got the data from
  81. self._parsers = {
  82. variant: [] for variant in self._override_order
  83. } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
  84. self._config = {
  85. variant: {} for variant in self._override_order
  86. } # type: Dict[Kind, Dict[str, Any]]
  87. self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]]
  88. def load(self):
  89. # type: () -> None
  90. """Loads configuration from configuration files and environment
  91. """
  92. self._load_config_files()
  93. if not self.isolated:
  94. self._load_environment_vars()
  95. def get_file_to_edit(self):
  96. # type: () -> Optional[str]
  97. """Returns the file with highest priority in configuration
  98. """
  99. assert self.load_only is not None, \
  100. "Need to be specified a file to be editing"
  101. try:
  102. return self._get_parser_to_modify()[0]
  103. except IndexError:
  104. return None
  105. def items(self):
  106. # type: () -> Iterable[Tuple[str, Any]]
  107. """Returns key-value pairs like dict.items() representing the loaded
  108. configuration
  109. """
  110. return self._dictionary.items()
  111. def get_value(self, key):
  112. # type: (str) -> Any
  113. """Get a value from the configuration.
  114. """
  115. try:
  116. return self._dictionary[key]
  117. except KeyError:
  118. raise ConfigurationError("No such key - {}".format(key))
  119. def set_value(self, key, value):
  120. # type: (str, Any) -> None
  121. """Modify a value in the configuration.
  122. """
  123. self._ensure_have_load_only()
  124. fname, parser = self._get_parser_to_modify()
  125. if parser is not None:
  126. section, name = _disassemble_key(key)
  127. # Modify the parser and the configuration
  128. if not parser.has_section(section):
  129. parser.add_section(section)
  130. parser.set(section, name, value)
  131. self._config[self.load_only][key] = value
  132. self._mark_as_modified(fname, parser)
  133. def unset_value(self, key):
  134. # type: (str) -> None
  135. """Unset a value in the configuration.
  136. """
  137. self._ensure_have_load_only()
  138. if key not in self._config[self.load_only]:
  139. raise ConfigurationError("No such key - {}".format(key))
  140. fname, parser = self._get_parser_to_modify()
  141. if parser is not None:
  142. section, name = _disassemble_key(key)
  143. # Remove the key in the parser
  144. modified_something = False
  145. if parser.has_section(section):
  146. # Returns whether the option was removed or not
  147. modified_something = parser.remove_option(section, name)
  148. if modified_something:
  149. # name removed from parser, section may now be empty
  150. section_iter = iter(parser.items(section))
  151. try:
  152. val = six.next(section_iter)
  153. except StopIteration:
  154. val = None
  155. if val is None:
  156. parser.remove_section(section)
  157. self._mark_as_modified(fname, parser)
  158. else:
  159. raise ConfigurationError(
  160. "Fatal Internal error [id=1]. Please report as a bug."
  161. )
  162. del self._config[self.load_only][key]
  163. def save(self):
  164. # type: () -> None
  165. """Save the currentin-memory state.
  166. """
  167. self._ensure_have_load_only()
  168. for fname, parser in self._modified_parsers:
  169. logger.info("Writing to %s", fname)
  170. # Ensure directory exists.
  171. ensure_dir(os.path.dirname(fname))
  172. with open(fname, "w") as f:
  173. parser.write(f) # type: ignore
  174. #
  175. # Private routines
  176. #
  177. def _ensure_have_load_only(self):
  178. # type: () -> None
  179. if self.load_only is None:
  180. raise ConfigurationError("Needed a specific file to be modifying.")
  181. logger.debug("Will be working with %s variant only", self.load_only)
  182. @property
  183. def _dictionary(self):
  184. # type: () -> Dict[str, Any]
  185. """A dictionary representing the loaded configuration.
  186. """
  187. # NOTE: Dictionaries are not populated if not loaded. So, conditionals
  188. # are not needed here.
  189. retval = {}
  190. for variant in self._override_order:
  191. retval.update(self._config[variant])
  192. return retval
  193. def _load_config_files(self):
  194. # type: () -> None
  195. """Loads configuration from configuration files
  196. """
  197. config_files = dict(self._iter_config_files())
  198. if config_files[kinds.ENV][0:1] == [os.devnull]:
  199. logger.debug(
  200. "Skipping loading configuration files due to "
  201. "environment's PIP_CONFIG_FILE being os.devnull"
  202. )
  203. return
  204. for variant, files in config_files.items():
  205. for fname in files:
  206. # If there's specific variant set in `load_only`, load only
  207. # that variant, not the others.
  208. if self.load_only is not None and variant != self.load_only:
  209. logger.debug(
  210. "Skipping file '%s' (variant: %s)", fname, variant
  211. )
  212. continue
  213. parser = self._load_file(variant, fname)
  214. # Keeping track of the parsers used
  215. self._parsers[variant].append((fname, parser))
  216. def _load_file(self, variant, fname):
  217. # type: (Kind, str) -> RawConfigParser
  218. logger.debug("For variant '%s', will try loading '%s'", variant, fname)
  219. parser = self._construct_parser(fname)
  220. for section in parser.sections():
  221. items = parser.items(section)
  222. self._config[variant].update(self._normalized_keys(section, items))
  223. return parser
  224. def _construct_parser(self, fname):
  225. # type: (str) -> RawConfigParser
  226. parser = configparser.RawConfigParser()
  227. # If there is no such file, don't bother reading it but create the
  228. # parser anyway, to hold the data.
  229. # Doing this is useful when modifying and saving files, where we don't
  230. # need to construct a parser.
  231. if os.path.exists(fname):
  232. try:
  233. parser.read(fname)
  234. except UnicodeDecodeError:
  235. # See https://github.com/pypa/pip/issues/4963
  236. raise ConfigurationFileCouldNotBeLoaded(
  237. reason="contains invalid {} characters".format(
  238. locale.getpreferredencoding(False)
  239. ),
  240. fname=fname,
  241. )
  242. except configparser.Error as error:
  243. # See https://github.com/pypa/pip/issues/4893
  244. raise ConfigurationFileCouldNotBeLoaded(error=error)
  245. return parser
  246. def _load_environment_vars(self):
  247. # type: () -> None
  248. """Loads configuration from environment variables
  249. """
  250. self._config[kinds.ENV_VAR].update(
  251. self._normalized_keys(":env:", self._get_environ_vars())
  252. )
  253. def _normalized_keys(self, section, items):
  254. # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
  255. """Normalizes items to construct a dictionary with normalized keys.
  256. This routine is where the names become keys and are made the same
  257. regardless of source - configuration files or environment.
  258. """
  259. normalized = {}
  260. for name, val in items:
  261. key = section + "." + _normalize_name(name)
  262. normalized[key] = val
  263. return normalized
  264. def _get_environ_vars(self):
  265. # type: () -> Iterable[Tuple[str, str]]
  266. """Returns a generator with all environmental vars with prefix PIP_"""
  267. for key, val in os.environ.items():
  268. should_be_yielded = (
  269. key.startswith("PIP_") and
  270. key[4:].lower() not in self._ignore_env_names
  271. )
  272. if should_be_yielded:
  273. yield key[4:].lower(), val
  274. # XXX: This is patched in the tests.
  275. def _iter_config_files(self):
  276. # type: () -> Iterable[Tuple[Kind, List[str]]]
  277. """Yields variant and configuration files associated with it.
  278. This should be treated like items of a dictionary.
  279. """
  280. # SMELL: Move the conditions out of this function
  281. # environment variables have the lowest priority
  282. config_file = os.environ.get('PIP_CONFIG_FILE', None)
  283. if config_file is not None:
  284. yield kinds.ENV, [config_file]
  285. else:
  286. yield kinds.ENV, []
  287. # at the base we have any global configuration
  288. yield kinds.GLOBAL, list(site_config_files)
  289. # per-user configuration next
  290. should_load_user_config = not self.isolated and not (
  291. config_file and os.path.exists(config_file)
  292. )
  293. if should_load_user_config:
  294. # The legacy config file is overridden by the new config file
  295. yield kinds.USER, [legacy_config_file, new_config_file]
  296. # finally virtualenv configuration first trumping others
  297. if running_under_virtualenv():
  298. yield kinds.VENV, [venv_config_file]
  299. def _get_parser_to_modify(self):
  300. # type: () -> Tuple[str, RawConfigParser]
  301. # Determine which parser to modify
  302. parsers = self._parsers[self.load_only]
  303. if not parsers:
  304. # This should not happen if everything works correctly.
  305. raise ConfigurationError(
  306. "Fatal Internal error [id=2]. Please report as a bug."
  307. )
  308. # Use the highest priority parser.
  309. return parsers[-1]
  310. # XXX: This is patched in the tests.
  311. def _mark_as_modified(self, fname, parser):
  312. # type: (str, RawConfigParser) -> None
  313. file_parser_tuple = (fname, parser)
  314. if file_parser_tuple not in self._modified_parsers:
  315. self._modified_parsers.append(file_parser_tuple)