Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

config.py 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. from __future__ import absolute_import, unicode_literals
  2. import io
  3. import os
  4. import sys
  5. from collections import defaultdict
  6. from functools import partial
  7. from importlib import import_module
  8. from distutils.errors import DistutilsOptionError, DistutilsFileError
  9. from setuptools.extern.packaging.version import LegacyVersion, parse
  10. from setuptools.extern.six import string_types, PY3
  11. __metaclass__ = type
  12. def read_configuration(
  13. filepath, find_others=False, ignore_option_errors=False):
  14. """Read given configuration file and returns options from it as a dict.
  15. :param str|unicode filepath: Path to configuration file
  16. to get options from.
  17. :param bool find_others: Whether to search for other configuration files
  18. which could be on in various places.
  19. :param bool ignore_option_errors: Whether to silently ignore
  20. options, values of which could not be resolved (e.g. due to exceptions
  21. in directives such as file:, attr:, etc.).
  22. If False exceptions are propagated as expected.
  23. :rtype: dict
  24. """
  25. from setuptools.dist import Distribution, _Distribution
  26. filepath = os.path.abspath(filepath)
  27. if not os.path.isfile(filepath):
  28. raise DistutilsFileError(
  29. 'Configuration file %s does not exist.' % filepath)
  30. current_directory = os.getcwd()
  31. os.chdir(os.path.dirname(filepath))
  32. try:
  33. dist = Distribution()
  34. filenames = dist.find_config_files() if find_others else []
  35. if filepath not in filenames:
  36. filenames.append(filepath)
  37. _Distribution.parse_config_files(dist, filenames=filenames)
  38. handlers = parse_configuration(
  39. dist, dist.command_options,
  40. ignore_option_errors=ignore_option_errors)
  41. finally:
  42. os.chdir(current_directory)
  43. return configuration_to_dict(handlers)
  44. def configuration_to_dict(handlers):
  45. """Returns configuration data gathered by given handlers as a dict.
  46. :param list[ConfigHandler] handlers: Handlers list,
  47. usually from parse_configuration()
  48. :rtype: dict
  49. """
  50. config_dict = defaultdict(dict)
  51. for handler in handlers:
  52. obj_alias = handler.section_prefix
  53. target_obj = handler.target_obj
  54. for option in handler.set_options:
  55. getter = getattr(target_obj, 'get_%s' % option, None)
  56. if getter is None:
  57. value = getattr(target_obj, option)
  58. else:
  59. value = getter()
  60. config_dict[obj_alias][option] = value
  61. return config_dict
  62. def parse_configuration(
  63. distribution, command_options, ignore_option_errors=False):
  64. """Performs additional parsing of configuration options
  65. for a distribution.
  66. Returns a list of used option handlers.
  67. :param Distribution distribution:
  68. :param dict command_options:
  69. :param bool ignore_option_errors: Whether to silently ignore
  70. options, values of which could not be resolved (e.g. due to exceptions
  71. in directives such as file:, attr:, etc.).
  72. If False exceptions are propagated as expected.
  73. :rtype: list
  74. """
  75. options = ConfigOptionsHandler(
  76. distribution, command_options, ignore_option_errors)
  77. options.parse()
  78. meta = ConfigMetadataHandler(
  79. distribution.metadata, command_options, ignore_option_errors, distribution.package_dir)
  80. meta.parse()
  81. return meta, options
  82. class ConfigHandler:
  83. """Handles metadata supplied in configuration files."""
  84. section_prefix = None
  85. """Prefix for config sections handled by this handler.
  86. Must be provided by class heirs.
  87. """
  88. aliases = {}
  89. """Options aliases.
  90. For compatibility with various packages. E.g.: d2to1 and pbr.
  91. Note: `-` in keys is replaced with `_` by config parser.
  92. """
  93. def __init__(self, target_obj, options, ignore_option_errors=False):
  94. sections = {}
  95. section_prefix = self.section_prefix
  96. for section_name, section_options in options.items():
  97. if not section_name.startswith(section_prefix):
  98. continue
  99. section_name = section_name.replace(section_prefix, '').strip('.')
  100. sections[section_name] = section_options
  101. self.ignore_option_errors = ignore_option_errors
  102. self.target_obj = target_obj
  103. self.sections = sections
  104. self.set_options = []
  105. @property
  106. def parsers(self):
  107. """Metadata item name to parser function mapping."""
  108. raise NotImplementedError(
  109. '%s must provide .parsers property' % self.__class__.__name__)
  110. def __setitem__(self, option_name, value):
  111. unknown = tuple()
  112. target_obj = self.target_obj
  113. # Translate alias into real name.
  114. option_name = self.aliases.get(option_name, option_name)
  115. current_value = getattr(target_obj, option_name, unknown)
  116. if current_value is unknown:
  117. raise KeyError(option_name)
  118. if current_value:
  119. # Already inhabited. Skipping.
  120. return
  121. skip_option = False
  122. parser = self.parsers.get(option_name)
  123. if parser:
  124. try:
  125. value = parser(value)
  126. except Exception:
  127. skip_option = True
  128. if not self.ignore_option_errors:
  129. raise
  130. if skip_option:
  131. return
  132. setter = getattr(target_obj, 'set_%s' % option_name, None)
  133. if setter is None:
  134. setattr(target_obj, option_name, value)
  135. else:
  136. setter(value)
  137. self.set_options.append(option_name)
  138. @classmethod
  139. def _parse_list(cls, value, separator=','):
  140. """Represents value as a list.
  141. Value is split either by separator (defaults to comma) or by lines.
  142. :param value:
  143. :param separator: List items separator character.
  144. :rtype: list
  145. """
  146. if isinstance(value, list): # _get_parser_compound case
  147. return value
  148. if '\n' in value:
  149. value = value.splitlines()
  150. else:
  151. value = value.split(separator)
  152. return [chunk.strip() for chunk in value if chunk.strip()]
  153. @classmethod
  154. def _parse_dict(cls, value):
  155. """Represents value as a dict.
  156. :param value:
  157. :rtype: dict
  158. """
  159. separator = '='
  160. result = {}
  161. for line in cls._parse_list(value):
  162. key, sep, val = line.partition(separator)
  163. if sep != separator:
  164. raise DistutilsOptionError(
  165. 'Unable to parse option value to dict: %s' % value)
  166. result[key.strip()] = val.strip()
  167. return result
  168. @classmethod
  169. def _parse_bool(cls, value):
  170. """Represents value as boolean.
  171. :param value:
  172. :rtype: bool
  173. """
  174. value = value.lower()
  175. return value in ('1', 'true', 'yes')
  176. @classmethod
  177. def _parse_file(cls, value):
  178. """Represents value as a string, allowing including text
  179. from nearest files using `file:` directive.
  180. Directive is sandboxed and won't reach anything outside
  181. directory with setup.py.
  182. Examples:
  183. file: LICENSE
  184. file: README.rst, CHANGELOG.md, src/file.txt
  185. :param str value:
  186. :rtype: str
  187. """
  188. include_directive = 'file:'
  189. if not isinstance(value, string_types):
  190. return value
  191. if not value.startswith(include_directive):
  192. return value
  193. spec = value[len(include_directive):]
  194. filepaths = (os.path.abspath(path.strip()) for path in spec.split(','))
  195. return '\n'.join(
  196. cls._read_file(path)
  197. for path in filepaths
  198. if (cls._assert_local(path) or True)
  199. and os.path.isfile(path)
  200. )
  201. @staticmethod
  202. def _assert_local(filepath):
  203. if not filepath.startswith(os.getcwd()):
  204. raise DistutilsOptionError(
  205. '`file:` directive can not access %s' % filepath)
  206. @staticmethod
  207. def _read_file(filepath):
  208. with io.open(filepath, encoding='utf-8') as f:
  209. return f.read()
  210. @classmethod
  211. def _parse_attr(cls, value, package_dir=None):
  212. """Represents value as a module attribute.
  213. Examples:
  214. attr: package.attr
  215. attr: package.module.attr
  216. :param str value:
  217. :rtype: str
  218. """
  219. attr_directive = 'attr:'
  220. if not value.startswith(attr_directive):
  221. return value
  222. attrs_path = value.replace(attr_directive, '').strip().split('.')
  223. attr_name = attrs_path.pop()
  224. module_name = '.'.join(attrs_path)
  225. module_name = module_name or '__init__'
  226. parent_path = os.getcwd()
  227. if package_dir:
  228. if attrs_path[0] in package_dir:
  229. # A custom path was specified for the module we want to import
  230. custom_path = package_dir[attrs_path[0]]
  231. parts = custom_path.rsplit('/', 1)
  232. if len(parts) > 1:
  233. parent_path = os.path.join(os.getcwd(), parts[0])
  234. module_name = parts[1]
  235. else:
  236. module_name = custom_path
  237. elif '' in package_dir:
  238. # A custom parent directory was specified for all root modules
  239. parent_path = os.path.join(os.getcwd(), package_dir[''])
  240. sys.path.insert(0, parent_path)
  241. try:
  242. module = import_module(module_name)
  243. value = getattr(module, attr_name)
  244. finally:
  245. sys.path = sys.path[1:]
  246. return value
  247. @classmethod
  248. def _get_parser_compound(cls, *parse_methods):
  249. """Returns parser function to represents value as a list.
  250. Parses a value applying given methods one after another.
  251. :param parse_methods:
  252. :rtype: callable
  253. """
  254. def parse(value):
  255. parsed = value
  256. for method in parse_methods:
  257. parsed = method(parsed)
  258. return parsed
  259. return parse
  260. @classmethod
  261. def _parse_section_to_dict(cls, section_options, values_parser=None):
  262. """Parses section options into a dictionary.
  263. Optionally applies a given parser to values.
  264. :param dict section_options:
  265. :param callable values_parser:
  266. :rtype: dict
  267. """
  268. value = {}
  269. values_parser = values_parser or (lambda val: val)
  270. for key, (_, val) in section_options.items():
  271. value[key] = values_parser(val)
  272. return value
  273. def parse_section(self, section_options):
  274. """Parses configuration file section.
  275. :param dict section_options:
  276. """
  277. for (name, (_, value)) in section_options.items():
  278. try:
  279. self[name] = value
  280. except KeyError:
  281. pass # Keep silent for a new option may appear anytime.
  282. def parse(self):
  283. """Parses configuration file items from one
  284. or more related sections.
  285. """
  286. for section_name, section_options in self.sections.items():
  287. method_postfix = ''
  288. if section_name: # [section.option] variant
  289. method_postfix = '_%s' % section_name
  290. section_parser_method = getattr(
  291. self,
  292. # Dots in section names are tranlsated into dunderscores.
  293. ('parse_section%s' % method_postfix).replace('.', '__'),
  294. None)
  295. if section_parser_method is None:
  296. raise DistutilsOptionError(
  297. 'Unsupported distribution option section: [%s.%s]' % (
  298. self.section_prefix, section_name))
  299. section_parser_method(section_options)
  300. class ConfigMetadataHandler(ConfigHandler):
  301. section_prefix = 'metadata'
  302. aliases = {
  303. 'home_page': 'url',
  304. 'summary': 'description',
  305. 'classifier': 'classifiers',
  306. 'platform': 'platforms',
  307. }
  308. strict_mode = False
  309. """We need to keep it loose, to be partially compatible with
  310. `pbr` and `d2to1` packages which also uses `metadata` section.
  311. """
  312. def __init__(self, target_obj, options, ignore_option_errors=False,
  313. package_dir=None):
  314. super(ConfigMetadataHandler, self).__init__(target_obj, options,
  315. ignore_option_errors)
  316. self.package_dir = package_dir
  317. @property
  318. def parsers(self):
  319. """Metadata item name to parser function mapping."""
  320. parse_list = self._parse_list
  321. parse_file = self._parse_file
  322. parse_dict = self._parse_dict
  323. return {
  324. 'platforms': parse_list,
  325. 'keywords': parse_list,
  326. 'provides': parse_list,
  327. 'requires': parse_list,
  328. 'obsoletes': parse_list,
  329. 'classifiers': self._get_parser_compound(parse_file, parse_list),
  330. 'license': parse_file,
  331. 'description': parse_file,
  332. 'long_description': parse_file,
  333. 'version': self._parse_version,
  334. 'project_urls': parse_dict,
  335. }
  336. def _parse_version(self, value):
  337. """Parses `version` option value.
  338. :param value:
  339. :rtype: str
  340. """
  341. version = self._parse_file(value)
  342. if version != value:
  343. version = version.strip()
  344. # Be strict about versions loaded from file because it's easy to
  345. # accidentally include newlines and other unintended content
  346. if isinstance(parse(version), LegacyVersion):
  347. raise DistutilsOptionError('Version loaded from %s does not comply with PEP 440: %s' % (
  348. value, version
  349. ))
  350. return version
  351. version = self._parse_attr(value, self.package_dir)
  352. if callable(version):
  353. version = version()
  354. if not isinstance(version, string_types):
  355. if hasattr(version, '__iter__'):
  356. version = '.'.join(map(str, version))
  357. else:
  358. version = '%s' % version
  359. return version
  360. class ConfigOptionsHandler(ConfigHandler):
  361. section_prefix = 'options'
  362. @property
  363. def parsers(self):
  364. """Metadata item name to parser function mapping."""
  365. parse_list = self._parse_list
  366. parse_list_semicolon = partial(self._parse_list, separator=';')
  367. parse_bool = self._parse_bool
  368. parse_dict = self._parse_dict
  369. return {
  370. 'zip_safe': parse_bool,
  371. 'use_2to3': parse_bool,
  372. 'include_package_data': parse_bool,
  373. 'package_dir': parse_dict,
  374. 'use_2to3_fixers': parse_list,
  375. 'use_2to3_exclude_fixers': parse_list,
  376. 'convert_2to3_doctests': parse_list,
  377. 'scripts': parse_list,
  378. 'eager_resources': parse_list,
  379. 'dependency_links': parse_list,
  380. 'namespace_packages': parse_list,
  381. 'install_requires': parse_list_semicolon,
  382. 'setup_requires': parse_list_semicolon,
  383. 'tests_require': parse_list_semicolon,
  384. 'packages': self._parse_packages,
  385. 'entry_points': self._parse_file,
  386. 'py_modules': parse_list,
  387. }
  388. def _parse_packages(self, value):
  389. """Parses `packages` option value.
  390. :param value:
  391. :rtype: list
  392. """
  393. find_directives = ['find:', 'find_namespace:']
  394. trimmed_value = value.strip()
  395. if not trimmed_value in find_directives:
  396. return self._parse_list(value)
  397. findns = trimmed_value == find_directives[1]
  398. if findns and not PY3:
  399. raise DistutilsOptionError('find_namespace: directive is unsupported on Python < 3.3')
  400. # Read function arguments from a dedicated section.
  401. find_kwargs = self.parse_section_packages__find(
  402. self.sections.get('packages.find', {}))
  403. if findns:
  404. from setuptools import find_namespace_packages as find_packages
  405. else:
  406. from setuptools import find_packages
  407. return find_packages(**find_kwargs)
  408. def parse_section_packages__find(self, section_options):
  409. """Parses `packages.find` configuration file section.
  410. To be used in conjunction with _parse_packages().
  411. :param dict section_options:
  412. """
  413. section_data = self._parse_section_to_dict(
  414. section_options, self._parse_list)
  415. valid_keys = ['where', 'include', 'exclude']
  416. find_kwargs = dict(
  417. [(k, v) for k, v in section_data.items() if k in valid_keys and v])
  418. where = find_kwargs.get('where')
  419. if where is not None:
  420. find_kwargs['where'] = where[0] # cast list to single val
  421. return find_kwargs
  422. def parse_section_entry_points(self, section_options):
  423. """Parses `entry_points` configuration file section.
  424. :param dict section_options:
  425. """
  426. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  427. self['entry_points'] = parsed
  428. def _parse_package_data(self, section_options):
  429. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  430. root = parsed.get('*')
  431. if root:
  432. parsed[''] = root
  433. del parsed['*']
  434. return parsed
  435. def parse_section_package_data(self, section_options):
  436. """Parses `package_data` configuration file section.
  437. :param dict section_options:
  438. """
  439. self['package_data'] = self._parse_package_data(section_options)
  440. def parse_section_exclude_package_data(self, section_options):
  441. """Parses `exclude_package_data` configuration file section.
  442. :param dict section_options:
  443. """
  444. self['exclude_package_data'] = self._parse_package_data(
  445. section_options)
  446. def parse_section_extras_require(self, section_options):
  447. """Parses `extras_require` configuration file section.
  448. :param dict section_options:
  449. """
  450. parse_list = partial(self._parse_list, separator=';')
  451. self['extras_require'] = self._parse_section_to_dict(
  452. section_options, parse_list)