build_py.py 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232
  1. from glob import glob
  2. from distutils.util import convert_path
  3. import distutils.command.build_py as orig
  4. import os
  5. import fnmatch
  6. import textwrap
  7. import io
  8. import distutils.errors
  9. import itertools
  10. import stat
  11. from setuptools.extern.more_itertools import unique_everseen
  12. def make_writable(target):
  13. os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
  14. class build_py(orig.build_py):
  15. """Enhanced 'build_py' command that includes data files with packages
  16. The data files are specified via a 'package_data' argument to 'setup()'.
  17. See 'setuptools.dist.Distribution' for more details.
  18. Also, this version of the 'build_py' command allows you to specify both
  19. 'py_modules' and 'packages' in the same setup operation.
  20. """
  21. def finalize_options(self):
  22. orig.build_py.finalize_options(self)
  23. self.package_data = self.distribution.package_data
  24. self.exclude_package_data = self.distribution.exclude_package_data or {}
  25. if 'data_files' in self.__dict__:
  26. del self.__dict__['data_files']
  27. self.__updated_files = []
  28. def run(self):
  29. """Build modules, packages, and copy data files to build directory"""
  30. if not self.py_modules and not self.packages:
  31. return
  32. if self.py_modules:
  33. self.build_modules()
  34. if self.packages:
  35. self.build_packages()
  36. self.build_package_data()
  37. # Only compile actual .py files, using our base class' idea of what our
  38. # output files are.
  39. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
  40. def __getattr__(self, attr):
  41. "lazily compute data files"
  42. if attr == 'data_files':
  43. self.data_files = self._get_data_files()
  44. return self.data_files
  45. return orig.build_py.__getattr__(self, attr)
  46. def build_module(self, module, module_file, package):
  47. outfile, copied = orig.build_py.build_module(self, module, module_file, package)
  48. if copied:
  49. self.__updated_files.append(outfile)
  50. return outfile, copied
  51. def _get_data_files(self):
  52. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  53. self.analyze_manifest()
  54. return list(map(self._get_pkg_data_files, self.packages or ()))
  55. def _get_pkg_data_files(self, package):
  56. # Locate package source directory
  57. src_dir = self.get_package_dir(package)
  58. # Compute package build directory
  59. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  60. # Strip directory from globbed filenames
  61. filenames = [
  62. os.path.relpath(file, src_dir)
  63. for file in self.find_data_files(package, src_dir)
  64. ]
  65. return package, src_dir, build_dir, filenames
  66. def find_data_files(self, package, src_dir):
  67. """Return filenames for package's data files in 'src_dir'"""
  68. patterns = self._get_platform_patterns(
  69. self.package_data,
  70. package,
  71. src_dir,
  72. )
  73. globs_expanded = map(glob, patterns)
  74. # flatten the expanded globs into an iterable of matches
  75. globs_matches = itertools.chain.from_iterable(globs_expanded)
  76. glob_files = filter(os.path.isfile, globs_matches)
  77. files = itertools.chain(
  78. self.manifest_files.get(package, []),
  79. glob_files,
  80. )
  81. return self.exclude_data_files(package, src_dir, files)
  82. def build_package_data(self):
  83. """Copy data files into build directory"""
  84. for package, src_dir, build_dir, filenames in self.data_files:
  85. for filename in filenames:
  86. target = os.path.join(build_dir, filename)
  87. self.mkpath(os.path.dirname(target))
  88. srcfile = os.path.join(src_dir, filename)
  89. outf, copied = self.copy_file(srcfile, target)
  90. make_writable(target)
  91. srcfile = os.path.abspath(srcfile)
  92. def analyze_manifest(self):
  93. self.manifest_files = mf = {}
  94. if not self.distribution.include_package_data:
  95. return
  96. src_dirs = {}
  97. for package in self.packages or ():
  98. # Locate package source directory
  99. src_dirs[assert_relative(self.get_package_dir(package))] = package
  100. self.run_command('egg_info')
  101. ei_cmd = self.get_finalized_command('egg_info')
  102. for path in ei_cmd.filelist.files:
  103. d, f = os.path.split(assert_relative(path))
  104. prev = None
  105. oldf = f
  106. while d and d != prev and d not in src_dirs:
  107. prev = d
  108. d, df = os.path.split(d)
  109. f = os.path.join(df, f)
  110. if d in src_dirs:
  111. if path.endswith('.py') and f == oldf:
  112. continue # it's a module, not data
  113. mf.setdefault(src_dirs[d], []).append(path)
  114. def get_data_files(self):
  115. pass # Lazily compute data files in _get_data_files() function.
  116. def check_package(self, package, package_dir):
  117. """Check namespace packages' __init__ for declare_namespace"""
  118. try:
  119. return self.packages_checked[package]
  120. except KeyError:
  121. pass
  122. init_py = orig.build_py.check_package(self, package, package_dir)
  123. self.packages_checked[package] = init_py
  124. if not init_py or not self.distribution.namespace_packages:
  125. return init_py
  126. for pkg in self.distribution.namespace_packages:
  127. if pkg == package or pkg.startswith(package + '.'):
  128. break
  129. else:
  130. return init_py
  131. with io.open(init_py, 'rb') as f:
  132. contents = f.read()
  133. if b'declare_namespace' not in contents:
  134. raise distutils.errors.DistutilsError(
  135. "Namespace package problem: %s is a namespace package, but "
  136. "its\n__init__.py does not call declare_namespace()! Please "
  137. 'fix it.\n(See the setuptools manual under '
  138. '"Namespace Packages" for details.)\n"' % (package,)
  139. )
  140. return init_py
  141. def initialize_options(self):
  142. self.packages_checked = {}
  143. orig.build_py.initialize_options(self)
  144. def get_package_dir(self, package):
  145. res = orig.build_py.get_package_dir(self, package)
  146. if self.distribution.src_root is not None:
  147. return os.path.join(self.distribution.src_root, res)
  148. return res
  149. def exclude_data_files(self, package, src_dir, files):
  150. """Filter filenames for package's data files in 'src_dir'"""
  151. files = list(files)
  152. patterns = self._get_platform_patterns(
  153. self.exclude_package_data,
  154. package,
  155. src_dir,
  156. )
  157. match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)
  158. # flatten the groups of matches into an iterable of matches
  159. matches = itertools.chain.from_iterable(match_groups)
  160. bad = set(matches)
  161. keepers = (fn for fn in files if fn not in bad)
  162. # ditch dupes
  163. return list(unique_everseen(keepers))
  164. @staticmethod
  165. def _get_platform_patterns(spec, package, src_dir):
  166. """
  167. yield platform-specific path patterns (suitable for glob
  168. or fn_match) from a glob-based spec (such as
  169. self.package_data or self.exclude_package_data)
  170. matching package in src_dir.
  171. """
  172. raw_patterns = itertools.chain(
  173. spec.get('', []),
  174. spec.get(package, []),
  175. )
  176. return (
  177. # Each pattern has to be converted to a platform-specific path
  178. os.path.join(src_dir, convert_path(pattern))
  179. for pattern in raw_patterns
  180. )
  181. def assert_relative(path):
  182. if not os.path.isabs(path):
  183. return path
  184. from distutils.errors import DistutilsSetupError
  185. msg = (
  186. textwrap.dedent(
  187. """
  188. Error: setup script specifies an absolute path:
  189. %s
  190. setup() arguments must *always* be /-separated paths relative to the
  191. setup.py directory, *never* absolute paths.
  192. """
  193. ).lstrip()
  194. % path
  195. )
  196. raise DistutilsSetupError(msg)