package_index.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119
  1. """PyPI and direct package downloading"""
  2. import sys
  3. import os
  4. import re
  5. import io
  6. import shutil
  7. import socket
  8. import base64
  9. import hashlib
  10. import itertools
  11. import warnings
  12. import configparser
  13. import html
  14. import http.client
  15. import urllib.parse
  16. import urllib.request
  17. import urllib.error
  18. from functools import wraps
  19. import setuptools
  20. from pkg_resources import (
  21. CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
  22. Environment, find_distributions, safe_name, safe_version,
  23. to_filename, Requirement, DEVELOP_DIST, EGG_DIST,
  24. )
  25. from distutils import log
  26. from distutils.errors import DistutilsError
  27. from fnmatch import translate
  28. from setuptools.wheel import Wheel
  29. from setuptools.extern.more_itertools import unique_everseen
  30. EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')
  31. HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
  32. PYPI_MD5 = re.compile(
  33. r'<a href="([^"#]+)">([^<]+)</a>\n\s+\(<a (?:title="MD5 hash"\n\s+)'
  34. r'href="[^?]+\?:action=show_md5&amp;digest=([0-9a-f]{32})">md5</a>\)'
  35. )
  36. URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match
  37. EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
  38. __all__ = [
  39. 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
  40. 'interpret_distro_name',
  41. ]
  42. _SOCKET_TIMEOUT = 15
  43. _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}"
  44. user_agent = _tmpl.format(
  45. py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools)
  46. def parse_requirement_arg(spec):
  47. try:
  48. return Requirement.parse(spec)
  49. except ValueError as e:
  50. raise DistutilsError(
  51. "Not a URL, existing file, or requirement spec: %r" % (spec,)
  52. ) from e
  53. def parse_bdist_wininst(name):
  54. """Return (base,pyversion) or (None,None) for possible .exe name"""
  55. lower = name.lower()
  56. base, py_ver, plat = None, None, None
  57. if lower.endswith('.exe'):
  58. if lower.endswith('.win32.exe'):
  59. base = name[:-10]
  60. plat = 'win32'
  61. elif lower.startswith('.win32-py', -16):
  62. py_ver = name[-7:-4]
  63. base = name[:-16]
  64. plat = 'win32'
  65. elif lower.endswith('.win-amd64.exe'):
  66. base = name[:-14]
  67. plat = 'win-amd64'
  68. elif lower.startswith('.win-amd64-py', -20):
  69. py_ver = name[-7:-4]
  70. base = name[:-20]
  71. plat = 'win-amd64'
  72. return base, py_ver, plat
  73. def egg_info_for_url(url):
  74. parts = urllib.parse.urlparse(url)
  75. scheme, server, path, parameters, query, fragment = parts
  76. base = urllib.parse.unquote(path.split('/')[-1])
  77. if server == 'sourceforge.net' and base == 'download': # XXX Yuck
  78. base = urllib.parse.unquote(path.split('/')[-2])
  79. if '#' in base:
  80. base, fragment = base.split('#', 1)
  81. return base, fragment
  82. def distros_for_url(url, metadata=None):
  83. """Yield egg or source distribution objects that might be found at a URL"""
  84. base, fragment = egg_info_for_url(url)
  85. for dist in distros_for_location(url, base, metadata):
  86. yield dist
  87. if fragment:
  88. match = EGG_FRAGMENT.match(fragment)
  89. if match:
  90. for dist in interpret_distro_name(
  91. url, match.group(1), metadata, precedence=CHECKOUT_DIST
  92. ):
  93. yield dist
  94. def distros_for_location(location, basename, metadata=None):
  95. """Yield egg or source distribution objects based on basename"""
  96. if basename.endswith('.egg.zip'):
  97. basename = basename[:-4] # strip the .zip
  98. if basename.endswith('.egg') and '-' in basename:
  99. # only one, unambiguous interpretation
  100. return [Distribution.from_location(location, basename, metadata)]
  101. if basename.endswith('.whl') and '-' in basename:
  102. wheel = Wheel(basename)
  103. if not wheel.is_compatible():
  104. return []
  105. return [Distribution(
  106. location=location,
  107. project_name=wheel.project_name,
  108. version=wheel.version,
  109. # Increase priority over eggs.
  110. precedence=EGG_DIST + 1,
  111. )]
  112. if basename.endswith('.exe'):
  113. win_base, py_ver, platform = parse_bdist_wininst(basename)
  114. if win_base is not None:
  115. return interpret_distro_name(
  116. location, win_base, metadata, py_ver, BINARY_DIST, platform
  117. )
  118. # Try source distro extensions (.zip, .tgz, etc.)
  119. #
  120. for ext in EXTENSIONS:
  121. if basename.endswith(ext):
  122. basename = basename[:-len(ext)]
  123. return interpret_distro_name(location, basename, metadata)
  124. return [] # no extension matched
  125. def distros_for_filename(filename, metadata=None):
  126. """Yield possible egg or source distribution objects based on a filename"""
  127. return distros_for_location(
  128. normalize_path(filename), os.path.basename(filename), metadata
  129. )
  130. def interpret_distro_name(
  131. location, basename, metadata, py_version=None, precedence=SOURCE_DIST,
  132. platform=None
  133. ):
  134. """Generate alternative interpretations of a source distro name
  135. Note: if `location` is a filesystem filename, you should call
  136. ``pkg_resources.normalize_path()`` on it before passing it to this
  137. routine!
  138. """
  139. # Generate alternative interpretations of a source distro name
  140. # Because some packages are ambiguous as to name/versions split
  141. # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
  142. # So, we generate each possible interpretation (e.g. "adns, python-1.1.0"
  143. # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice,
  144. # the spurious interpretations should be ignored, because in the event
  145. # there's also an "adns" package, the spurious "python-1.1.0" version will
  146. # compare lower than any numeric version number, and is therefore unlikely
  147. # to match a request for it. It's still a potential problem, though, and
  148. # in the long run PyPI and the distutils should go for "safe" names and
  149. # versions in distribution archive names (sdist and bdist).
  150. parts = basename.split('-')
  151. if not py_version and any(re.match(r'py\d\.\d$', p) for p in parts[2:]):
  152. # it is a bdist_dumb, not an sdist -- bail out
  153. return
  154. for p in range(1, len(parts) + 1):
  155. yield Distribution(
  156. location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
  157. py_version=py_version, precedence=precedence,
  158. platform=platform
  159. )
  160. def unique_values(func):
  161. """
  162. Wrap a function returning an iterable such that the resulting iterable
  163. only ever yields unique items.
  164. """
  165. @wraps(func)
  166. def wrapper(*args, **kwargs):
  167. return unique_everseen(func(*args, **kwargs))
  168. return wrapper
  169. REL = re.compile(r"""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I)
  170. # this line is here to fix emacs' cruddy broken syntax highlighting
  171. @unique_values
  172. def find_external_links(url, page):
  173. """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""
  174. for match in REL.finditer(page):
  175. tag, rel = match.groups()
  176. rels = set(map(str.strip, rel.lower().split(',')))
  177. if 'homepage' in rels or 'download' in rels:
  178. for match in HREF.finditer(tag):
  179. yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
  180. for tag in ("<th>Home Page", "<th>Download URL"):
  181. pos = page.find(tag)
  182. if pos != -1:
  183. match = HREF.search(page, pos)
  184. if match:
  185. yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
  186. class ContentChecker:
  187. """
  188. A null content checker that defines the interface for checking content
  189. """
  190. def feed(self, block):
  191. """
  192. Feed a block of data to the hash.
  193. """
  194. return
  195. def is_valid(self):
  196. """
  197. Check the hash. Return False if validation fails.
  198. """
  199. return True
  200. def report(self, reporter, template):
  201. """
  202. Call reporter with information about the checker (hash name)
  203. substituted into the template.
  204. """
  205. return
  206. class HashChecker(ContentChecker):
  207. pattern = re.compile(
  208. r'(?P<hash_name>sha1|sha224|sha384|sha256|sha512|md5)='
  209. r'(?P<expected>[a-f0-9]+)'
  210. )
  211. def __init__(self, hash_name, expected):
  212. self.hash_name = hash_name
  213. self.hash = hashlib.new(hash_name)
  214. self.expected = expected
  215. @classmethod
  216. def from_url(cls, url):
  217. "Construct a (possibly null) ContentChecker from a URL"
  218. fragment = urllib.parse.urlparse(url)[-1]
  219. if not fragment:
  220. return ContentChecker()
  221. match = cls.pattern.search(fragment)
  222. if not match:
  223. return ContentChecker()
  224. return cls(**match.groupdict())
  225. def feed(self, block):
  226. self.hash.update(block)
  227. def is_valid(self):
  228. return self.hash.hexdigest() == self.expected
  229. def report(self, reporter, template):
  230. msg = template % self.hash_name
  231. return reporter(msg)
  232. class PackageIndex(Environment):
  233. """A distribution index that scans web pages for download URLs"""
  234. def __init__(
  235. self, index_url="https://pypi.org/simple/", hosts=('*',),
  236. ca_bundle=None, verify_ssl=True, *args, **kw
  237. ):
  238. Environment.__init__(self, *args, **kw)
  239. self.index_url = index_url + "/" [:not index_url.endswith('/')]
  240. self.scanned_urls = {}
  241. self.fetched_urls = {}
  242. self.package_pages = {}
  243. self.allows = re.compile('|'.join(map(translate, hosts))).match
  244. self.to_scan = []
  245. self.opener = urllib.request.urlopen
  246. # FIXME: 'PackageIndex.process_url' is too complex (14)
  247. def process_url(self, url, retrieve=False): # noqa: C901
  248. """Evaluate a URL as a possible download, and maybe retrieve it"""
  249. if url in self.scanned_urls and not retrieve:
  250. return
  251. self.scanned_urls[url] = True
  252. if not URL_SCHEME(url):
  253. self.process_filename(url)
  254. return
  255. else:
  256. dists = list(distros_for_url(url))
  257. if dists:
  258. if not self.url_ok(url):
  259. return
  260. self.debug("Found link: %s", url)
  261. if dists or not retrieve or url in self.fetched_urls:
  262. list(map(self.add, dists))
  263. return # don't need the actual page
  264. if not self.url_ok(url):
  265. self.fetched_urls[url] = True
  266. return
  267. self.info("Reading %s", url)
  268. self.fetched_urls[url] = True # prevent multiple fetch attempts
  269. tmpl = "Download error on %s: %%s -- Some packages may not be found!"
  270. f = self.open_url(url, tmpl % url)
  271. if f is None:
  272. return
  273. if isinstance(f, urllib.error.HTTPError) and f.code == 401:
  274. self.info("Authentication error: %s" % f.msg)
  275. self.fetched_urls[f.url] = True
  276. if 'html' not in f.headers.get('content-type', '').lower():
  277. f.close() # not html, we can't process it
  278. return
  279. base = f.url # handle redirects
  280. page = f.read()
  281. if not isinstance(page, str):
  282. # In Python 3 and got bytes but want str.
  283. if isinstance(f, urllib.error.HTTPError):
  284. # Errors have no charset, assume latin1:
  285. charset = 'latin-1'
  286. else:
  287. charset = f.headers.get_param('charset') or 'latin-1'
  288. page = page.decode(charset, "ignore")
  289. f.close()
  290. for match in HREF.finditer(page):
  291. link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
  292. self.process_url(link)
  293. if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
  294. page = self.process_index(url, page)
  295. def process_filename(self, fn, nested=False):
  296. # process filenames or directories
  297. if not os.path.exists(fn):
  298. self.warn("Not found: %s", fn)
  299. return
  300. if os.path.isdir(fn) and not nested:
  301. path = os.path.realpath(fn)
  302. for item in os.listdir(path):
  303. self.process_filename(os.path.join(path, item), True)
  304. dists = distros_for_filename(fn)
  305. if dists:
  306. self.debug("Found: %s", fn)
  307. list(map(self.add, dists))
  308. def url_ok(self, url, fatal=False):
  309. s = URL_SCHEME(url)
  310. is_file = s and s.group(1).lower() == 'file'
  311. if is_file or self.allows(urllib.parse.urlparse(url)[1]):
  312. return True
  313. msg = (
  314. "\nNote: Bypassing %s (disallowed host; see "
  315. "http://bit.ly/2hrImnY for details).\n")
  316. if fatal:
  317. raise DistutilsError(msg % url)
  318. else:
  319. self.warn(msg, url)
  320. def scan_egg_links(self, search_path):
  321. dirs = filter(os.path.isdir, search_path)
  322. egg_links = (
  323. (path, entry)
  324. for path in dirs
  325. for entry in os.listdir(path)
  326. if entry.endswith('.egg-link')
  327. )
  328. list(itertools.starmap(self.scan_egg_link, egg_links))
  329. def scan_egg_link(self, path, entry):
  330. with open(os.path.join(path, entry)) as raw_lines:
  331. # filter non-empty lines
  332. lines = list(filter(None, map(str.strip, raw_lines)))
  333. if len(lines) != 2:
  334. # format is not recognized; punt
  335. return
  336. egg_path, setup_path = lines
  337. for dist in find_distributions(os.path.join(path, egg_path)):
  338. dist.location = os.path.join(path, *lines)
  339. dist.precedence = SOURCE_DIST
  340. self.add(dist)
  341. def _scan(self, link):
  342. # Process a URL to see if it's for a package page
  343. NO_MATCH_SENTINEL = None, None
  344. if not link.startswith(self.index_url):
  345. return NO_MATCH_SENTINEL
  346. parts = list(map(
  347. urllib.parse.unquote, link[len(self.index_url):].split('/')
  348. ))
  349. if len(parts) != 2 or '#' in parts[1]:
  350. return NO_MATCH_SENTINEL
  351. # it's a package page, sanitize and index it
  352. pkg = safe_name(parts[0])
  353. ver = safe_version(parts[1])
  354. self.package_pages.setdefault(pkg.lower(), {})[link] = True
  355. return to_filename(pkg), to_filename(ver)
  356. def process_index(self, url, page):
  357. """Process the contents of a PyPI page"""
  358. # process an index page into the package-page index
  359. for match in HREF.finditer(page):
  360. try:
  361. self._scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))
  362. except ValueError:
  363. pass
  364. pkg, ver = self._scan(url) # ensure this page is in the page index
  365. if not pkg:
  366. return "" # no sense double-scanning non-package pages
  367. # process individual package page
  368. for new_url in find_external_links(url, page):
  369. # Process the found URL
  370. base, frag = egg_info_for_url(new_url)
  371. if base.endswith('.py') and not frag:
  372. if ver:
  373. new_url += '#egg=%s-%s' % (pkg, ver)
  374. else:
  375. self.need_version_info(url)
  376. self.scan_url(new_url)
  377. return PYPI_MD5.sub(
  378. lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1, 3, 2), page
  379. )
  380. def need_version_info(self, url):
  381. self.scan_all(
  382. "Page at %s links to .py file(s) without version info; an index "
  383. "scan is required.", url
  384. )
  385. def scan_all(self, msg=None, *args):
  386. if self.index_url not in self.fetched_urls:
  387. if msg:
  388. self.warn(msg, *args)
  389. self.info(
  390. "Scanning index of all packages (this may take a while)"
  391. )
  392. self.scan_url(self.index_url)
  393. def find_packages(self, requirement):
  394. self.scan_url(self.index_url + requirement.unsafe_name + '/')
  395. if not self.package_pages.get(requirement.key):
  396. # Fall back to safe version of the name
  397. self.scan_url(self.index_url + requirement.project_name + '/')
  398. if not self.package_pages.get(requirement.key):
  399. # We couldn't find the target package, so search the index page too
  400. self.not_found_in_index(requirement)
  401. for url in list(self.package_pages.get(requirement.key, ())):
  402. # scan each page that might be related to the desired package
  403. self.scan_url(url)
  404. def obtain(self, requirement, installer=None):
  405. self.prescan()
  406. self.find_packages(requirement)
  407. for dist in self[requirement.key]:
  408. if dist in requirement:
  409. return dist
  410. self.debug("%s does not match %s", requirement, dist)
  411. return super(PackageIndex, self).obtain(requirement, installer)
  412. def check_hash(self, checker, filename, tfp):
  413. """
  414. checker is a ContentChecker
  415. """
  416. checker.report(
  417. self.debug,
  418. "Validating %%s checksum for %s" % filename)
  419. if not checker.is_valid():
  420. tfp.close()
  421. os.unlink(filename)
  422. raise DistutilsError(
  423. "%s validation failed for %s; "
  424. "possible download problem?"
  425. % (checker.hash.name, os.path.basename(filename))
  426. )
  427. def add_find_links(self, urls):
  428. """Add `urls` to the list that will be prescanned for searches"""
  429. for url in urls:
  430. if (
  431. self.to_scan is None # if we have already "gone online"
  432. or not URL_SCHEME(url) # or it's a local file/directory
  433. or url.startswith('file:')
  434. or list(distros_for_url(url)) # or a direct package link
  435. ):
  436. # then go ahead and process it now
  437. self.scan_url(url)
  438. else:
  439. # otherwise, defer retrieval till later
  440. self.to_scan.append(url)
  441. def prescan(self):
  442. """Scan urls scheduled for prescanning (e.g. --find-links)"""
  443. if self.to_scan:
  444. list(map(self.scan_url, self.to_scan))
  445. self.to_scan = None # from now on, go ahead and process immediately
  446. def not_found_in_index(self, requirement):
  447. if self[requirement.key]: # we've seen at least one distro
  448. meth, msg = self.info, "Couldn't retrieve index page for %r"
  449. else: # no distros seen for this name, might be misspelled
  450. meth, msg = (
  451. self.warn,
  452. "Couldn't find index page for %r (maybe misspelled?)")
  453. meth(msg, requirement.unsafe_name)
  454. self.scan_all()
  455. def download(self, spec, tmpdir):
  456. """Locate and/or download `spec` to `tmpdir`, returning a local path
  457. `spec` may be a ``Requirement`` object, or a string containing a URL,
  458. an existing local filename, or a project/version requirement spec
  459. (i.e. the string form of a ``Requirement`` object). If it is the URL
  460. of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
  461. that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
  462. automatically created alongside the downloaded file.
  463. If `spec` is a ``Requirement`` object or a string containing a
  464. project/version requirement spec, this method returns the location of
  465. a matching distribution (possibly after downloading it to `tmpdir`).
  466. If `spec` is a locally existing file or directory name, it is simply
  467. returned unchanged. If `spec` is a URL, it is downloaded to a subpath
  468. of `tmpdir`, and the local filename is returned. Various errors may be
  469. raised if a problem occurs during downloading.
  470. """
  471. if not isinstance(spec, Requirement):
  472. scheme = URL_SCHEME(spec)
  473. if scheme:
  474. # It's a url, download it to tmpdir
  475. found = self._download_url(scheme.group(1), spec, tmpdir)
  476. base, fragment = egg_info_for_url(spec)
  477. if base.endswith('.py'):
  478. found = self.gen_setup(found, fragment, tmpdir)
  479. return found
  480. elif os.path.exists(spec):
  481. # Existing file or directory, just return it
  482. return spec
  483. else:
  484. spec = parse_requirement_arg(spec)
  485. return getattr(self.fetch_distribution(spec, tmpdir), 'location', None)
  486. def fetch_distribution( # noqa: C901 # is too complex (14) # FIXME
  487. self, requirement, tmpdir, force_scan=False, source=False,
  488. develop_ok=False, local_index=None):
  489. """Obtain a distribution suitable for fulfilling `requirement`
  490. `requirement` must be a ``pkg_resources.Requirement`` instance.
  491. If necessary, or if the `force_scan` flag is set, the requirement is
  492. searched for in the (online) package index as well as the locally
  493. installed packages. If a distribution matching `requirement` is found,
  494. the returned distribution's ``location`` is the value you would have
  495. gotten from calling the ``download()`` method with the matching
  496. distribution's URL or filename. If no matching distribution is found,
  497. ``None`` is returned.
  498. If the `source` flag is set, only source distributions and source
  499. checkout links will be considered. Unless the `develop_ok` flag is
  500. set, development and system eggs (i.e., those using the ``.egg-info``
  501. format) will be ignored.
  502. """
  503. # process a Requirement
  504. self.info("Searching for %s", requirement)
  505. skipped = {}
  506. dist = None
  507. def find(req, env=None):
  508. if env is None:
  509. env = self
  510. # Find a matching distribution; may be called more than once
  511. for dist in env[req.key]:
  512. if dist.precedence == DEVELOP_DIST and not develop_ok:
  513. if dist not in skipped:
  514. self.warn(
  515. "Skipping development or system egg: %s", dist,
  516. )
  517. skipped[dist] = 1
  518. continue
  519. test = (
  520. dist in req
  521. and (dist.precedence <= SOURCE_DIST or not source)
  522. )
  523. if test:
  524. loc = self.download(dist.location, tmpdir)
  525. dist.download_location = loc
  526. if os.path.exists(dist.download_location):
  527. return dist
  528. if force_scan:
  529. self.prescan()
  530. self.find_packages(requirement)
  531. dist = find(requirement)
  532. if not dist and local_index is not None:
  533. dist = find(requirement, local_index)
  534. if dist is None:
  535. if self.to_scan is not None:
  536. self.prescan()
  537. dist = find(requirement)
  538. if dist is None and not force_scan:
  539. self.find_packages(requirement)
  540. dist = find(requirement)
  541. if dist is None:
  542. self.warn(
  543. "No local packages or working download links found for %s%s",
  544. (source and "a source distribution of " or ""),
  545. requirement,
  546. )
  547. else:
  548. self.info("Best match: %s", dist)
  549. return dist.clone(location=dist.download_location)
  550. def fetch(self, requirement, tmpdir, force_scan=False, source=False):
  551. """Obtain a file suitable for fulfilling `requirement`
  552. DEPRECATED; use the ``fetch_distribution()`` method now instead. For
  553. backward compatibility, this routine is identical but returns the
  554. ``location`` of the downloaded distribution instead of a distribution
  555. object.
  556. """
  557. dist = self.fetch_distribution(requirement, tmpdir, force_scan, source)
  558. if dist is not None:
  559. return dist.location
  560. return None
  561. def gen_setup(self, filename, fragment, tmpdir):
  562. match = EGG_FRAGMENT.match(fragment)
  563. dists = match and [
  564. d for d in
  565. interpret_distro_name(filename, match.group(1), None) if d.version
  566. ] or []
  567. if len(dists) == 1: # unambiguous ``#egg`` fragment
  568. basename = os.path.basename(filename)
  569. # Make sure the file has been downloaded to the temp dir.
  570. if os.path.dirname(filename) != tmpdir:
  571. dst = os.path.join(tmpdir, basename)
  572. from setuptools.command.easy_install import samefile
  573. if not samefile(filename, dst):
  574. shutil.copy2(filename, dst)
  575. filename = dst
  576. with open(os.path.join(tmpdir, 'setup.py'), 'w') as file:
  577. file.write(
  578. "from setuptools import setup\n"
  579. "setup(name=%r, version=%r, py_modules=[%r])\n"
  580. % (
  581. dists[0].project_name, dists[0].version,
  582. os.path.splitext(basename)[0]
  583. )
  584. )
  585. return filename
  586. elif match:
  587. raise DistutilsError(
  588. "Can't unambiguously interpret project/version identifier %r; "
  589. "any dashes in the name or version should be escaped using "
  590. "underscores. %r" % (fragment, dists)
  591. )
  592. else:
  593. raise DistutilsError(
  594. "Can't process plain .py files without an '#egg=name-version'"
  595. " suffix to enable automatic setup script generation."
  596. )
  597. dl_blocksize = 8192
  598. def _download_to(self, url, filename):
  599. self.info("Downloading %s", url)
  600. # Download the file
  601. fp = None
  602. try:
  603. checker = HashChecker.from_url(url)
  604. fp = self.open_url(url)
  605. if isinstance(fp, urllib.error.HTTPError):
  606. raise DistutilsError(
  607. "Can't download %s: %s %s" % (url, fp.code, fp.msg)
  608. )
  609. headers = fp.info()
  610. blocknum = 0
  611. bs = self.dl_blocksize
  612. size = -1
  613. if "content-length" in headers:
  614. # Some servers return multiple Content-Length headers :(
  615. sizes = headers.get_all('Content-Length')
  616. size = max(map(int, sizes))
  617. self.reporthook(url, filename, blocknum, bs, size)
  618. with open(filename, 'wb') as tfp:
  619. while True:
  620. block = fp.read(bs)
  621. if block:
  622. checker.feed(block)
  623. tfp.write(block)
  624. blocknum += 1
  625. self.reporthook(url, filename, blocknum, bs, size)
  626. else:
  627. break
  628. self.check_hash(checker, filename, tfp)
  629. return headers
  630. finally:
  631. if fp:
  632. fp.close()
  633. def reporthook(self, url, filename, blocknum, blksize, size):
  634. pass # no-op
  635. # FIXME:
  636. def open_url(self, url, warning=None): # noqa: C901 # is too complex (12)
  637. if url.startswith('file:'):
  638. return local_open(url)
  639. try:
  640. return open_with_auth(url, self.opener)
  641. except (ValueError, http.client.InvalidURL) as v:
  642. msg = ' '.join([str(arg) for arg in v.args])
  643. if warning:
  644. self.warn(warning, msg)
  645. else:
  646. raise DistutilsError('%s %s' % (url, msg)) from v
  647. except urllib.error.HTTPError as v:
  648. return v
  649. except urllib.error.URLError as v:
  650. if warning:
  651. self.warn(warning, v.reason)
  652. else:
  653. raise DistutilsError("Download error for %s: %s"
  654. % (url, v.reason)) from v
  655. except http.client.BadStatusLine as v:
  656. if warning:
  657. self.warn(warning, v.line)
  658. else:
  659. raise DistutilsError(
  660. '%s returned a bad status line. The server might be '
  661. 'down, %s' %
  662. (url, v.line)
  663. ) from v
  664. except (http.client.HTTPException, socket.error) as v:
  665. if warning:
  666. self.warn(warning, v)
  667. else:
  668. raise DistutilsError("Download error for %s: %s"
  669. % (url, v)) from v
  670. def _download_url(self, scheme, url, tmpdir):
  671. # Determine download filename
  672. #
  673. name, fragment = egg_info_for_url(url)
  674. if name:
  675. while '..' in name:
  676. name = name.replace('..', '.').replace('\\', '_')
  677. else:
  678. name = "__downloaded__" # default if URL has no path contents
  679. if name.endswith('.egg.zip'):
  680. name = name[:-4] # strip the extra .zip before download
  681. filename = os.path.join(tmpdir, name)
  682. # Download the file
  683. #
  684. if scheme == 'svn' or scheme.startswith('svn+'):
  685. return self._download_svn(url, filename)
  686. elif scheme == 'git' or scheme.startswith('git+'):
  687. return self._download_git(url, filename)
  688. elif scheme.startswith('hg+'):
  689. return self._download_hg(url, filename)
  690. elif scheme == 'file':
  691. return urllib.request.url2pathname(urllib.parse.urlparse(url)[2])
  692. else:
  693. self.url_ok(url, True) # raises error if not allowed
  694. return self._attempt_download(url, filename)
  695. def scan_url(self, url):
  696. self.process_url(url, True)
  697. def _attempt_download(self, url, filename):
  698. headers = self._download_to(url, filename)
  699. if 'html' in headers.get('content-type', '').lower():
  700. return self._download_html(url, headers, filename)
  701. else:
  702. return filename
  703. def _download_html(self, url, headers, filename):
  704. file = open(filename)
  705. for line in file:
  706. if line.strip():
  707. # Check for a subversion index page
  708. if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
  709. # it's a subversion index page:
  710. file.close()
  711. os.unlink(filename)
  712. return self._download_svn(url, filename)
  713. break # not an index page
  714. file.close()
  715. os.unlink(filename)
  716. raise DistutilsError("Unexpected HTML page found at " + url)
  717. def _download_svn(self, url, filename):
  718. warnings.warn("SVN download support is deprecated", UserWarning)
  719. url = url.split('#', 1)[0] # remove any fragment for svn's sake
  720. creds = ''
  721. if url.lower().startswith('svn:') and '@' in url:
  722. scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
  723. if not netloc and path.startswith('//') and '/' in path[2:]:
  724. netloc, path = path[2:].split('/', 1)
  725. auth, host = _splituser(netloc)
  726. if auth:
  727. if ':' in auth:
  728. user, pw = auth.split(':', 1)
  729. creds = " --username=%s --password=%s" % (user, pw)
  730. else:
  731. creds = " --username=" + auth
  732. netloc = host
  733. parts = scheme, netloc, url, p, q, f
  734. url = urllib.parse.urlunparse(parts)
  735. self.info("Doing subversion checkout from %s to %s", url, filename)
  736. os.system("svn checkout%s -q %s %s" % (creds, url, filename))
  737. return filename
  738. @staticmethod
  739. def _vcs_split_rev_from_url(url, pop_prefix=False):
  740. scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
  741. scheme = scheme.split('+', 1)[-1]
  742. # Some fragment identification fails
  743. path = path.split('#', 1)[0]
  744. rev = None
  745. if '@' in path:
  746. path, rev = path.rsplit('@', 1)
  747. # Also, discard fragment
  748. url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))
  749. return url, rev
  750. def _download_git(self, url, filename):
  751. filename = filename.split('#', 1)[0]
  752. url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
  753. self.info("Doing git clone from %s to %s", url, filename)
  754. os.system("git clone --quiet %s %s" % (url, filename))
  755. if rev is not None:
  756. self.info("Checking out %s", rev)
  757. os.system("git -C %s checkout --quiet %s" % (
  758. filename,
  759. rev,
  760. ))
  761. return filename
  762. def _download_hg(self, url, filename):
  763. filename = filename.split('#', 1)[0]
  764. url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
  765. self.info("Doing hg clone from %s to %s", url, filename)
  766. os.system("hg clone --quiet %s %s" % (url, filename))
  767. if rev is not None:
  768. self.info("Updating to %s", rev)
  769. os.system("hg --cwd %s up -C -r %s -q" % (
  770. filename,
  771. rev,
  772. ))
  773. return filename
  774. def debug(self, msg, *args):
  775. log.debug(msg, *args)
  776. def info(self, msg, *args):
  777. log.info(msg, *args)
  778. def warn(self, msg, *args):
  779. log.warn(msg, *args)
  780. # This pattern matches a character entity reference (a decimal numeric
  781. # references, a hexadecimal numeric reference, or a named reference).
  782. entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub
  783. def decode_entity(match):
  784. what = match.group(0)
  785. return html.unescape(what)
  786. def htmldecode(text):
  787. """
  788. Decode HTML entities in the given text.
  789. >>> htmldecode(
  790. ... 'https://../package_name-0.1.2.tar.gz'
  791. ... '?tokena=A&amp;tokenb=B">package_name-0.1.2.tar.gz')
  792. 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
  793. """
  794. return entity_sub(decode_entity, text)
  795. def socket_timeout(timeout=15):
  796. def _socket_timeout(func):
  797. def _socket_timeout(*args, **kwargs):
  798. old_timeout = socket.getdefaulttimeout()
  799. socket.setdefaulttimeout(timeout)
  800. try:
  801. return func(*args, **kwargs)
  802. finally:
  803. socket.setdefaulttimeout(old_timeout)
  804. return _socket_timeout
  805. return _socket_timeout
  806. def _encode_auth(auth):
  807. """
  808. Encode auth from a URL suitable for an HTTP header.
  809. >>> str(_encode_auth('username%3Apassword'))
  810. 'dXNlcm5hbWU6cGFzc3dvcmQ='
  811. Long auth strings should not cause a newline to be inserted.
  812. >>> long_auth = 'username:' + 'password'*10
  813. >>> chr(10) in str(_encode_auth(long_auth))
  814. False
  815. """
  816. auth_s = urllib.parse.unquote(auth)
  817. # convert to bytes
  818. auth_bytes = auth_s.encode()
  819. encoded_bytes = base64.b64encode(auth_bytes)
  820. # convert back to a string
  821. encoded = encoded_bytes.decode()
  822. # strip the trailing carriage return
  823. return encoded.replace('\n', '')
  824. class Credential:
  825. """
  826. A username/password pair. Use like a namedtuple.
  827. """
  828. def __init__(self, username, password):
  829. self.username = username
  830. self.password = password
  831. def __iter__(self):
  832. yield self.username
  833. yield self.password
  834. def __str__(self):
  835. return '%(username)s:%(password)s' % vars(self)
  836. class PyPIConfig(configparser.RawConfigParser):
  837. def __init__(self):
  838. """
  839. Load from ~/.pypirc
  840. """
  841. defaults = dict.fromkeys(['username', 'password', 'repository'], '')
  842. configparser.RawConfigParser.__init__(self, defaults)
  843. rc = os.path.join(os.path.expanduser('~'), '.pypirc')
  844. if os.path.exists(rc):
  845. self.read(rc)
  846. @property
  847. def creds_by_repository(self):
  848. sections_with_repositories = [
  849. section for section in self.sections()
  850. if self.get(section, 'repository').strip()
  851. ]
  852. return dict(map(self._get_repo_cred, sections_with_repositories))
  853. def _get_repo_cred(self, section):
  854. repo = self.get(section, 'repository').strip()
  855. return repo, Credential(
  856. self.get(section, 'username').strip(),
  857. self.get(section, 'password').strip(),
  858. )
  859. def find_credential(self, url):
  860. """
  861. If the URL indicated appears to be a repository defined in this
  862. config, return the credential for that repository.
  863. """
  864. for repository, cred in self.creds_by_repository.items():
  865. if url.startswith(repository):
  866. return cred
  867. def open_with_auth(url, opener=urllib.request.urlopen):
  868. """Open a urllib2 request, handling HTTP authentication"""
  869. parsed = urllib.parse.urlparse(url)
  870. scheme, netloc, path, params, query, frag = parsed
  871. # Double scheme does not raise on macOS as revealed by a
  872. # failing test. We would expect "nonnumeric port". Refs #20.
  873. if netloc.endswith(':'):
  874. raise http.client.InvalidURL("nonnumeric port: ''")
  875. if scheme in ('http', 'https'):
  876. auth, address = _splituser(netloc)
  877. else:
  878. auth = None
  879. if not auth:
  880. cred = PyPIConfig().find_credential(url)
  881. if cred:
  882. auth = str(cred)
  883. info = cred.username, url
  884. log.info('Authenticating as %s for %s (from .pypirc)', *info)
  885. if auth:
  886. auth = "Basic " + _encode_auth(auth)
  887. parts = scheme, address, path, params, query, frag
  888. new_url = urllib.parse.urlunparse(parts)
  889. request = urllib.request.Request(new_url)
  890. request.add_header("Authorization", auth)
  891. else:
  892. request = urllib.request.Request(url)
  893. request.add_header('User-Agent', user_agent)
  894. fp = opener(request)
  895. if auth:
  896. # Put authentication info back into request URL if same host,
  897. # so that links found on the page will work
  898. s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url)
  899. if s2 == scheme and h2 == address:
  900. parts = s2, netloc, path2, param2, query2, frag2
  901. fp.url = urllib.parse.urlunparse(parts)
  902. return fp
  903. # copy of urllib.parse._splituser from Python 3.8
  904. def _splituser(host):
  905. """splituser('user[:passwd]@host[:port]')
  906. --> 'user[:passwd]', 'host[:port]'."""
  907. user, delim, host = host.rpartition('@')
  908. return (user if delim else None), host
  909. # adding a timeout to avoid freezing package_index
  910. open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)
  911. def fix_sf_url(url):
  912. return url # backward compatibility
  913. def local_open(url):
  914. """Read a local path, with special support for directories"""
  915. scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
  916. filename = urllib.request.url2pathname(path)
  917. if os.path.isfile(filename):
  918. return urllib.request.urlopen(url)
  919. elif path.endswith('/') and os.path.isdir(filename):
  920. files = []
  921. for f in os.listdir(filename):
  922. filepath = os.path.join(filename, f)
  923. if f == 'index.html':
  924. with open(filepath, 'r') as fp:
  925. body = fp.read()
  926. break
  927. elif os.path.isdir(filepath):
  928. f += '/'
  929. files.append('<a href="{name}">{name}</a>'.format(name=f))
  930. else:
  931. tmpl = (
  932. "<html><head><title>{url}</title>"
  933. "</head><body>{files}</body></html>")
  934. body = tmpl.format(url=url, files='\n'.join(files))
  935. status, message = 200, "OK"
  936. else:
  937. status, message, body = 404, "Path not found", "Not found"
  938. headers = {'content-type': 'text/html'}
  939. body_stream = io.StringIO(body)
  940. return urllib.error.HTTPError(url, status, message, headers, body_stream)