setupcfg.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689
  1. """Load setuptools configuration from ``setup.cfg`` files"""
  2. import os
  3. import warnings
  4. import functools
  5. from collections import defaultdict
  6. from functools import partial
  7. from functools import wraps
  8. from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List,
  9. Optional, Tuple, TypeVar, Union)
  10. from distutils.errors import DistutilsOptionError, DistutilsFileError
  11. from setuptools.extern.packaging.version import Version, InvalidVersion
  12. from setuptools.extern.packaging.specifiers import SpecifierSet
  13. from . import expand
  14. if TYPE_CHECKING:
  15. from setuptools.dist import Distribution # noqa
  16. from distutils.dist import DistributionMetadata # noqa
  17. _Path = Union[str, os.PathLike]
  18. SingleCommandOptions = Dict["str", Tuple["str", Any]]
  19. """Dict that associate the name of the options of a particular command to a
  20. tuple. The first element of the tuple indicates the origin of the option value
  21. (e.g. the name of the configuration file where it was read from),
  22. while the second element of the tuple is the option value itself
  23. """
  24. AllCommandOptions = Dict["str", SingleCommandOptions] # cmd name => its options
  25. Target = TypeVar("Target", bound=Union["Distribution", "DistributionMetadata"])
  26. def read_configuration(
  27. filepath: _Path,
  28. find_others=False,
  29. ignore_option_errors=False
  30. ) -> dict:
  31. """Read given configuration file and returns options from it as a dict.
  32. :param str|unicode filepath: Path to configuration file
  33. to get options from.
  34. :param bool find_others: Whether to search for other configuration files
  35. which could be on in various places.
  36. :param bool ignore_option_errors: Whether to silently ignore
  37. options, values of which could not be resolved (e.g. due to exceptions
  38. in directives such as file:, attr:, etc.).
  39. If False exceptions are propagated as expected.
  40. :rtype: dict
  41. """
  42. from setuptools.dist import Distribution
  43. dist = Distribution()
  44. filenames = dist.find_config_files() if find_others else []
  45. handlers = _apply(dist, filepath, filenames, ignore_option_errors)
  46. return configuration_to_dict(handlers)
  47. def apply_configuration(dist: "Distribution", filepath: _Path) -> "Distribution":
  48. """Apply the configuration from a ``setup.cfg`` file into an existing
  49. distribution object.
  50. """
  51. _apply(dist, filepath)
  52. dist._finalize_requires()
  53. return dist
  54. def _apply(
  55. dist: "Distribution", filepath: _Path,
  56. other_files: Iterable[_Path] = (),
  57. ignore_option_errors: bool = False,
  58. ) -> Tuple["ConfigHandler", ...]:
  59. """Read configuration from ``filepath`` and applies to the ``dist`` object."""
  60. from setuptools.dist import _Distribution
  61. filepath = os.path.abspath(filepath)
  62. if not os.path.isfile(filepath):
  63. raise DistutilsFileError('Configuration file %s does not exist.' % filepath)
  64. current_directory = os.getcwd()
  65. os.chdir(os.path.dirname(filepath))
  66. filenames = [*other_files, filepath]
  67. try:
  68. _Distribution.parse_config_files(dist, filenames=filenames)
  69. handlers = parse_configuration(
  70. dist, dist.command_options, ignore_option_errors=ignore_option_errors
  71. )
  72. dist._finalize_license_files()
  73. finally:
  74. os.chdir(current_directory)
  75. return handlers
  76. def _get_option(target_obj: Target, key: str):
  77. """
  78. Given a target object and option key, get that option from
  79. the target object, either through a get_{key} method or
  80. from an attribute directly.
  81. """
  82. getter_name = 'get_{key}'.format(**locals())
  83. by_attribute = functools.partial(getattr, target_obj, key)
  84. getter = getattr(target_obj, getter_name, by_attribute)
  85. return getter()
  86. def configuration_to_dict(handlers: Tuple["ConfigHandler", ...]) -> dict:
  87. """Returns configuration data gathered by given handlers as a dict.
  88. :param list[ConfigHandler] handlers: Handlers list,
  89. usually from parse_configuration()
  90. :rtype: dict
  91. """
  92. config_dict: dict = defaultdict(dict)
  93. for handler in handlers:
  94. for option in handler.set_options:
  95. value = _get_option(handler.target_obj, option)
  96. config_dict[handler.section_prefix][option] = value
  97. return config_dict
  98. def parse_configuration(
  99. distribution: "Distribution",
  100. command_options: AllCommandOptions,
  101. ignore_option_errors=False
  102. ) -> Tuple["ConfigMetadataHandler", "ConfigOptionsHandler"]:
  103. """Performs additional parsing of configuration options
  104. for a distribution.
  105. Returns a list of used option handlers.
  106. :param Distribution distribution:
  107. :param dict command_options:
  108. :param bool ignore_option_errors: Whether to silently ignore
  109. options, values of which could not be resolved (e.g. due to exceptions
  110. in directives such as file:, attr:, etc.).
  111. If False exceptions are propagated as expected.
  112. :rtype: list
  113. """
  114. with expand.EnsurePackagesDiscovered(distribution) as ensure_discovered:
  115. options = ConfigOptionsHandler(
  116. distribution,
  117. command_options,
  118. ignore_option_errors,
  119. ensure_discovered,
  120. )
  121. options.parse()
  122. if not distribution.package_dir:
  123. distribution.package_dir = options.package_dir # Filled by `find_packages`
  124. meta = ConfigMetadataHandler(
  125. distribution.metadata,
  126. command_options,
  127. ignore_option_errors,
  128. ensure_discovered,
  129. distribution.package_dir,
  130. distribution.src_root,
  131. )
  132. meta.parse()
  133. return meta, options
  134. class ConfigHandler(Generic[Target]):
  135. """Handles metadata supplied in configuration files."""
  136. section_prefix: str
  137. """Prefix for config sections handled by this handler.
  138. Must be provided by class heirs.
  139. """
  140. aliases: Dict[str, str] = {}
  141. """Options aliases.
  142. For compatibility with various packages. E.g.: d2to1 and pbr.
  143. Note: `-` in keys is replaced with `_` by config parser.
  144. """
  145. def __init__(
  146. self,
  147. target_obj: Target,
  148. options: AllCommandOptions,
  149. ignore_option_errors,
  150. ensure_discovered: expand.EnsurePackagesDiscovered,
  151. ):
  152. sections: AllCommandOptions = {}
  153. section_prefix = self.section_prefix
  154. for section_name, section_options in options.items():
  155. if not section_name.startswith(section_prefix):
  156. continue
  157. section_name = section_name.replace(section_prefix, '').strip('.')
  158. sections[section_name] = section_options
  159. self.ignore_option_errors = ignore_option_errors
  160. self.target_obj = target_obj
  161. self.sections = sections
  162. self.set_options: List[str] = []
  163. self.ensure_discovered = ensure_discovered
  164. @property
  165. def parsers(self):
  166. """Metadata item name to parser function mapping."""
  167. raise NotImplementedError(
  168. '%s must provide .parsers property' % self.__class__.__name__
  169. )
  170. def __setitem__(self, option_name, value):
  171. unknown = tuple()
  172. target_obj = self.target_obj
  173. # Translate alias into real name.
  174. option_name = self.aliases.get(option_name, option_name)
  175. current_value = getattr(target_obj, option_name, unknown)
  176. if current_value is unknown:
  177. raise KeyError(option_name)
  178. if current_value:
  179. # Already inhabited. Skipping.
  180. return
  181. skip_option = False
  182. parser = self.parsers.get(option_name)
  183. if parser:
  184. try:
  185. value = parser(value)
  186. except Exception:
  187. skip_option = True
  188. if not self.ignore_option_errors:
  189. raise
  190. if skip_option:
  191. return
  192. setter = getattr(target_obj, 'set_%s' % option_name, None)
  193. if setter is None:
  194. setattr(target_obj, option_name, value)
  195. else:
  196. setter(value)
  197. self.set_options.append(option_name)
  198. @classmethod
  199. def _parse_list(cls, value, separator=','):
  200. """Represents value as a list.
  201. Value is split either by separator (defaults to comma) or by lines.
  202. :param value:
  203. :param separator: List items separator character.
  204. :rtype: list
  205. """
  206. if isinstance(value, list): # _get_parser_compound case
  207. return value
  208. if '\n' in value:
  209. value = value.splitlines()
  210. else:
  211. value = value.split(separator)
  212. return [chunk.strip() for chunk in value if chunk.strip()]
  213. @classmethod
  214. def _parse_dict(cls, value):
  215. """Represents value as a dict.
  216. :param value:
  217. :rtype: dict
  218. """
  219. separator = '='
  220. result = {}
  221. for line in cls._parse_list(value):
  222. key, sep, val = line.partition(separator)
  223. if sep != separator:
  224. raise DistutilsOptionError(
  225. 'Unable to parse option value to dict: %s' % value
  226. )
  227. result[key.strip()] = val.strip()
  228. return result
  229. @classmethod
  230. def _parse_bool(cls, value):
  231. """Represents value as boolean.
  232. :param value:
  233. :rtype: bool
  234. """
  235. value = value.lower()
  236. return value in ('1', 'true', 'yes')
  237. @classmethod
  238. def _exclude_files_parser(cls, key):
  239. """Returns a parser function to make sure field inputs
  240. are not files.
  241. Parses a value after getting the key so error messages are
  242. more informative.
  243. :param key:
  244. :rtype: callable
  245. """
  246. def parser(value):
  247. exclude_directive = 'file:'
  248. if value.startswith(exclude_directive):
  249. raise ValueError(
  250. 'Only strings are accepted for the {0} field, '
  251. 'files are not accepted'.format(key)
  252. )
  253. return value
  254. return parser
  255. @classmethod
  256. def _parse_file(cls, value, root_dir: _Path):
  257. """Represents value as a string, allowing including text
  258. from nearest files using `file:` directive.
  259. Directive is sandboxed and won't reach anything outside
  260. directory with setup.py.
  261. Examples:
  262. file: README.rst, CHANGELOG.md, src/file.txt
  263. :param str value:
  264. :rtype: str
  265. """
  266. include_directive = 'file:'
  267. if not isinstance(value, str):
  268. return value
  269. if not value.startswith(include_directive):
  270. return value
  271. spec = value[len(include_directive) :]
  272. filepaths = (path.strip() for path in spec.split(','))
  273. return expand.read_files(filepaths, root_dir)
  274. def _parse_attr(self, value, package_dir, root_dir: _Path):
  275. """Represents value as a module attribute.
  276. Examples:
  277. attr: package.attr
  278. attr: package.module.attr
  279. :param str value:
  280. :rtype: str
  281. """
  282. attr_directive = 'attr:'
  283. if not value.startswith(attr_directive):
  284. return value
  285. attr_desc = value.replace(attr_directive, '')
  286. # Make sure package_dir is populated correctly, so `attr:` directives can work
  287. package_dir.update(self.ensure_discovered.package_dir)
  288. return expand.read_attr(attr_desc, package_dir, root_dir)
  289. @classmethod
  290. def _get_parser_compound(cls, *parse_methods):
  291. """Returns parser function to represents value as a list.
  292. Parses a value applying given methods one after another.
  293. :param parse_methods:
  294. :rtype: callable
  295. """
  296. def parse(value):
  297. parsed = value
  298. for method in parse_methods:
  299. parsed = method(parsed)
  300. return parsed
  301. return parse
  302. @classmethod
  303. def _parse_section_to_dict(cls, section_options, values_parser=None):
  304. """Parses section options into a dictionary.
  305. Optionally applies a given parser to values.
  306. :param dict section_options:
  307. :param callable values_parser:
  308. :rtype: dict
  309. """
  310. value = {}
  311. values_parser = values_parser or (lambda val: val)
  312. for key, (_, val) in section_options.items():
  313. value[key] = values_parser(val)
  314. return value
  315. def parse_section(self, section_options):
  316. """Parses configuration file section.
  317. :param dict section_options:
  318. """
  319. for (name, (_, value)) in section_options.items():
  320. try:
  321. self[name] = value
  322. except KeyError:
  323. pass # Keep silent for a new option may appear anytime.
  324. def parse(self):
  325. """Parses configuration file items from one
  326. or more related sections.
  327. """
  328. for section_name, section_options in self.sections.items():
  329. method_postfix = ''
  330. if section_name: # [section.option] variant
  331. method_postfix = '_%s' % section_name
  332. section_parser_method: Optional[Callable] = getattr(
  333. self,
  334. # Dots in section names are translated into dunderscores.
  335. ('parse_section%s' % method_postfix).replace('.', '__'),
  336. None,
  337. )
  338. if section_parser_method is None:
  339. raise DistutilsOptionError(
  340. 'Unsupported distribution option section: [%s.%s]'
  341. % (self.section_prefix, section_name)
  342. )
  343. section_parser_method(section_options)
  344. def _deprecated_config_handler(self, func, msg, warning_class):
  345. """this function will wrap around parameters that are deprecated
  346. :param msg: deprecation message
  347. :param warning_class: class of warning exception to be raised
  348. :param func: function to be wrapped around
  349. """
  350. @wraps(func)
  351. def config_handler(*args, **kwargs):
  352. warnings.warn(msg, warning_class)
  353. return func(*args, **kwargs)
  354. return config_handler
  355. class ConfigMetadataHandler(ConfigHandler["DistributionMetadata"]):
  356. section_prefix = 'metadata'
  357. aliases = {
  358. 'home_page': 'url',
  359. 'summary': 'description',
  360. 'classifier': 'classifiers',
  361. 'platform': 'platforms',
  362. }
  363. strict_mode = False
  364. """We need to keep it loose, to be partially compatible with
  365. `pbr` and `d2to1` packages which also uses `metadata` section.
  366. """
  367. def __init__(
  368. self,
  369. target_obj: "DistributionMetadata",
  370. options: AllCommandOptions,
  371. ignore_option_errors: bool,
  372. ensure_discovered: expand.EnsurePackagesDiscovered,
  373. package_dir: Optional[dict] = None,
  374. root_dir: _Path = os.curdir
  375. ):
  376. super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)
  377. self.package_dir = package_dir
  378. self.root_dir = root_dir
  379. @property
  380. def parsers(self):
  381. """Metadata item name to parser function mapping."""
  382. parse_list = self._parse_list
  383. parse_file = partial(self._parse_file, root_dir=self.root_dir)
  384. parse_dict = self._parse_dict
  385. exclude_files_parser = self._exclude_files_parser
  386. return {
  387. 'platforms': parse_list,
  388. 'keywords': parse_list,
  389. 'provides': parse_list,
  390. 'requires': self._deprecated_config_handler(
  391. parse_list,
  392. "The requires parameter is deprecated, please use "
  393. "install_requires for runtime dependencies.",
  394. DeprecationWarning,
  395. ),
  396. 'obsoletes': parse_list,
  397. 'classifiers': self._get_parser_compound(parse_file, parse_list),
  398. 'license': exclude_files_parser('license'),
  399. 'license_file': self._deprecated_config_handler(
  400. exclude_files_parser('license_file'),
  401. "The license_file parameter is deprecated, "
  402. "use license_files instead.",
  403. DeprecationWarning,
  404. ),
  405. 'license_files': parse_list,
  406. 'description': parse_file,
  407. 'long_description': parse_file,
  408. 'version': self._parse_version,
  409. 'project_urls': parse_dict,
  410. }
  411. def _parse_version(self, value):
  412. """Parses `version` option value.
  413. :param value:
  414. :rtype: str
  415. """
  416. version = self._parse_file(value, self.root_dir)
  417. if version != value:
  418. version = version.strip()
  419. # Be strict about versions loaded from file because it's easy to
  420. # accidentally include newlines and other unintended content
  421. try:
  422. Version(version)
  423. except InvalidVersion:
  424. tmpl = (
  425. 'Version loaded from {value} does not '
  426. 'comply with PEP 440: {version}'
  427. )
  428. raise DistutilsOptionError(tmpl.format(**locals()))
  429. return version
  430. return expand.version(self._parse_attr(value, self.package_dir, self.root_dir))
  431. class ConfigOptionsHandler(ConfigHandler["Distribution"]):
  432. section_prefix = 'options'
  433. def __init__(
  434. self,
  435. target_obj: "Distribution",
  436. options: AllCommandOptions,
  437. ignore_option_errors: bool,
  438. ensure_discovered: expand.EnsurePackagesDiscovered,
  439. ):
  440. super().__init__(target_obj, options, ignore_option_errors, ensure_discovered)
  441. self.root_dir = target_obj.src_root
  442. self.package_dir: Dict[str, str] = {} # To be filled by `find_packages`
  443. @property
  444. def parsers(self):
  445. """Metadata item name to parser function mapping."""
  446. parse_list = self._parse_list
  447. parse_list_semicolon = partial(self._parse_list, separator=';')
  448. parse_bool = self._parse_bool
  449. parse_dict = self._parse_dict
  450. parse_cmdclass = self._parse_cmdclass
  451. parse_file = partial(self._parse_file, root_dir=self.root_dir)
  452. return {
  453. 'zip_safe': parse_bool,
  454. 'include_package_data': parse_bool,
  455. 'package_dir': parse_dict,
  456. 'scripts': parse_list,
  457. 'eager_resources': parse_list,
  458. 'dependency_links': parse_list,
  459. 'namespace_packages': parse_list,
  460. 'install_requires': parse_list_semicolon,
  461. 'setup_requires': parse_list_semicolon,
  462. 'tests_require': parse_list_semicolon,
  463. 'packages': self._parse_packages,
  464. 'entry_points': parse_file,
  465. 'py_modules': parse_list,
  466. 'python_requires': SpecifierSet,
  467. 'cmdclass': parse_cmdclass,
  468. }
  469. def _parse_cmdclass(self, value):
  470. package_dir = self.ensure_discovered.package_dir
  471. return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir)
  472. def _parse_packages(self, value):
  473. """Parses `packages` option value.
  474. :param value:
  475. :rtype: list
  476. """
  477. find_directives = ['find:', 'find_namespace:']
  478. trimmed_value = value.strip()
  479. if trimmed_value not in find_directives:
  480. return self._parse_list(value)
  481. # Read function arguments from a dedicated section.
  482. find_kwargs = self.parse_section_packages__find(
  483. self.sections.get('packages.find', {})
  484. )
  485. find_kwargs.update(
  486. namespaces=(trimmed_value == find_directives[1]),
  487. root_dir=self.root_dir,
  488. fill_package_dir=self.package_dir,
  489. )
  490. return expand.find_packages(**find_kwargs)
  491. def parse_section_packages__find(self, section_options):
  492. """Parses `packages.find` configuration file section.
  493. To be used in conjunction with _parse_packages().
  494. :param dict section_options:
  495. """
  496. section_data = self._parse_section_to_dict(section_options, self._parse_list)
  497. valid_keys = ['where', 'include', 'exclude']
  498. find_kwargs = dict(
  499. [(k, v) for k, v in section_data.items() if k in valid_keys and v]
  500. )
  501. where = find_kwargs.get('where')
  502. if where is not None:
  503. find_kwargs['where'] = where[0] # cast list to single val
  504. return find_kwargs
  505. def parse_section_entry_points(self, section_options):
  506. """Parses `entry_points` configuration file section.
  507. :param dict section_options:
  508. """
  509. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  510. self['entry_points'] = parsed
  511. def _parse_package_data(self, section_options):
  512. package_data = self._parse_section_to_dict(section_options, self._parse_list)
  513. return expand.canonic_package_data(package_data)
  514. def parse_section_package_data(self, section_options):
  515. """Parses `package_data` configuration file section.
  516. :param dict section_options:
  517. """
  518. self['package_data'] = self._parse_package_data(section_options)
  519. def parse_section_exclude_package_data(self, section_options):
  520. """Parses `exclude_package_data` configuration file section.
  521. :param dict section_options:
  522. """
  523. self['exclude_package_data'] = self._parse_package_data(section_options)
  524. def parse_section_extras_require(self, section_options):
  525. """Parses `extras_require` configuration file section.
  526. :param dict section_options:
  527. """
  528. parse_list = partial(self._parse_list, separator=';')
  529. parsed = self._parse_section_to_dict(section_options, parse_list)
  530. self['extras_require'] = parsed
  531. def parse_section_data_files(self, section_options):
  532. """Parses `data_files` configuration file section.
  533. :param dict section_options:
  534. """
  535. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  536. self['data_files'] = expand.canonic_data_files(parsed, self.root_dir)