build_py.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. from glob import glob
  2. from distutils.util import convert_path
  3. import distutils.command.build_py as orig
  4. import os
  5. import fnmatch
  6. import textwrap
  7. import io
  8. import distutils.errors
  9. import itertools
  10. import stat
  11. from setuptools.extern.more_itertools import unique_everseen
  12. def make_writable(target):
  13. os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
  14. class build_py(orig.build_py):
  15. """Enhanced 'build_py' command that includes data files with packages
  16. The data files are specified via a 'package_data' argument to 'setup()'.
  17. See 'setuptools.dist.Distribution' for more details.
  18. Also, this version of the 'build_py' command allows you to specify both
  19. 'py_modules' and 'packages' in the same setup operation.
  20. """
  21. def finalize_options(self):
  22. orig.build_py.finalize_options(self)
  23. self.package_data = self.distribution.package_data
  24. self.exclude_package_data = self.distribution.exclude_package_data or {}
  25. if 'data_files' in self.__dict__:
  26. del self.__dict__['data_files']
  27. self.__updated_files = []
  28. def run(self):
  29. """Build modules, packages, and copy data files to build directory"""
  30. if not self.py_modules and not self.packages:
  31. return
  32. if self.py_modules:
  33. self.build_modules()
  34. if self.packages:
  35. self.build_packages()
  36. self.build_package_data()
  37. # Only compile actual .py files, using our base class' idea of what our
  38. # output files are.
  39. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
  40. def __getattr__(self, attr):
  41. "lazily compute data files"
  42. if attr == 'data_files':
  43. self.data_files = self._get_data_files()
  44. return self.data_files
  45. return orig.build_py.__getattr__(self, attr)
  46. def build_module(self, module, module_file, package):
  47. outfile, copied = orig.build_py.build_module(self, module, module_file, package)
  48. if copied:
  49. self.__updated_files.append(outfile)
  50. return outfile, copied
  51. def _get_data_files(self):
  52. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  53. self.analyze_manifest()
  54. return list(map(self._get_pkg_data_files, self.packages or ()))
  55. def get_data_files_without_manifest(self):
  56. """
  57. Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,
  58. but without triggering any attempt to analyze or build the manifest.
  59. """
  60. # Prevent eventual errors from unset `manifest_files`
  61. # (that would otherwise be set by `analyze_manifest`)
  62. self.__dict__.setdefault('manifest_files', {})
  63. return list(map(self._get_pkg_data_files, self.packages or ()))
  64. def _get_pkg_data_files(self, package):
  65. # Locate package source directory
  66. src_dir = self.get_package_dir(package)
  67. # Compute package build directory
  68. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  69. # Strip directory from globbed filenames
  70. filenames = [
  71. os.path.relpath(file, src_dir)
  72. for file in self.find_data_files(package, src_dir)
  73. ]
  74. return package, src_dir, build_dir, filenames
  75. def find_data_files(self, package, src_dir):
  76. """Return filenames for package's data files in 'src_dir'"""
  77. patterns = self._get_platform_patterns(
  78. self.package_data,
  79. package,
  80. src_dir,
  81. )
  82. globs_expanded = map(glob, patterns)
  83. # flatten the expanded globs into an iterable of matches
  84. globs_matches = itertools.chain.from_iterable(globs_expanded)
  85. glob_files = filter(os.path.isfile, globs_matches)
  86. files = itertools.chain(
  87. self.manifest_files.get(package, []),
  88. glob_files,
  89. )
  90. return self.exclude_data_files(package, src_dir, files)
  91. def build_package_data(self):
  92. """Copy data files into build directory"""
  93. for package, src_dir, build_dir, filenames in self.data_files:
  94. for filename in filenames:
  95. target = os.path.join(build_dir, filename)
  96. self.mkpath(os.path.dirname(target))
  97. srcfile = os.path.join(src_dir, filename)
  98. outf, copied = self.copy_file(srcfile, target)
  99. make_writable(target)
  100. srcfile = os.path.abspath(srcfile)
  101. def analyze_manifest(self):
  102. self.manifest_files = mf = {}
  103. if not self.distribution.include_package_data:
  104. return
  105. src_dirs = {}
  106. for package in self.packages or ():
  107. # Locate package source directory
  108. src_dirs[assert_relative(self.get_package_dir(package))] = package
  109. self.run_command('egg_info')
  110. ei_cmd = self.get_finalized_command('egg_info')
  111. for path in ei_cmd.filelist.files:
  112. d, f = os.path.split(assert_relative(path))
  113. prev = None
  114. oldf = f
  115. while d and d != prev and d not in src_dirs:
  116. prev = d
  117. d, df = os.path.split(d)
  118. f = os.path.join(df, f)
  119. if d in src_dirs:
  120. if path.endswith('.py') and f == oldf:
  121. continue # it's a module, not data
  122. mf.setdefault(src_dirs[d], []).append(path)
  123. def get_data_files(self):
  124. pass # Lazily compute data files in _get_data_files() function.
  125. def check_package(self, package, package_dir):
  126. """Check namespace packages' __init__ for declare_namespace"""
  127. try:
  128. return self.packages_checked[package]
  129. except KeyError:
  130. pass
  131. init_py = orig.build_py.check_package(self, package, package_dir)
  132. self.packages_checked[package] = init_py
  133. if not init_py or not self.distribution.namespace_packages:
  134. return init_py
  135. for pkg in self.distribution.namespace_packages:
  136. if pkg == package or pkg.startswith(package + '.'):
  137. break
  138. else:
  139. return init_py
  140. with io.open(init_py, 'rb') as f:
  141. contents = f.read()
  142. if b'declare_namespace' not in contents:
  143. raise distutils.errors.DistutilsError(
  144. "Namespace package problem: %s is a namespace package, but "
  145. "its\n__init__.py does not call declare_namespace()! Please "
  146. 'fix it.\n(See the setuptools manual under '
  147. '"Namespace Packages" for details.)\n"' % (package,)
  148. )
  149. return init_py
  150. def initialize_options(self):
  151. self.packages_checked = {}
  152. orig.build_py.initialize_options(self)
  153. def get_package_dir(self, package):
  154. res = orig.build_py.get_package_dir(self, package)
  155. if self.distribution.src_root is not None:
  156. return os.path.join(self.distribution.src_root, res)
  157. return res
  158. def exclude_data_files(self, package, src_dir, files):
  159. """Filter filenames for package's data files in 'src_dir'"""
  160. files = list(files)
  161. patterns = self._get_platform_patterns(
  162. self.exclude_package_data,
  163. package,
  164. src_dir,
  165. )
  166. match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)
  167. # flatten the groups of matches into an iterable of matches
  168. matches = itertools.chain.from_iterable(match_groups)
  169. bad = set(matches)
  170. keepers = (fn for fn in files if fn not in bad)
  171. # ditch dupes
  172. return list(unique_everseen(keepers))
  173. @staticmethod
  174. def _get_platform_patterns(spec, package, src_dir):
  175. """
  176. yield platform-specific path patterns (suitable for glob
  177. or fn_match) from a glob-based spec (such as
  178. self.package_data or self.exclude_package_data)
  179. matching package in src_dir.
  180. """
  181. raw_patterns = itertools.chain(
  182. spec.get('', []),
  183. spec.get(package, []),
  184. )
  185. return (
  186. # Each pattern has to be converted to a platform-specific path
  187. os.path.join(src_dir, convert_path(pattern))
  188. for pattern in raw_patterns
  189. )
  190. def assert_relative(path):
  191. if not os.path.isabs(path):
  192. return path
  193. from distutils.errors import DistutilsSetupError
  194. msg = (
  195. textwrap.dedent(
  196. """
  197. Error: setup script specifies an absolute path:
  198. %s
  199. setup() arguments must *always* be /-separated paths relative to the
  200. setup.py directory, *never* absolute paths.
  201. """
  202. ).lstrip()
  203. % path
  204. )
  205. raise DistutilsSetupError(msg)