preprocessor.js 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111
  1. 'use strict'
  2. const util = require('util')
  3. const fs = require('graceful-fs')
  4. // bind need only for mock unit tests
  5. const readFile = util.promisify(fs.readFile.bind(fs))
  6. const tryToRead = function (path, log) {
  7. const maxRetries = 3
  8. let promise = readFile(path)
  9. for (let retryCount = 1; retryCount <= maxRetries; retryCount++) {
  10. promise = promise.catch((err) => {
  11. log.warn(err)
  12. log.warn('retrying ' + retryCount)
  13. return readFile(path)
  14. })
  15. }
  16. return promise.catch((err) => {
  17. log.warn(err)
  18. return Promise.reject(err)
  19. })
  20. }
  21. const mm = require('minimatch')
  22. const { isBinaryFile } = require('isbinaryfile')
  23. const _ = require('lodash')
  24. const CryptoUtils = require('./utils/crypto-utils')
  25. const log = require('./logger').create('preprocess')
  26. function executeProcessor (process, file, content) {
  27. let done = null
  28. const donePromise = new Promise((resolve, reject) => {
  29. done = function (error, content) {
  30. // normalize B-C
  31. if (arguments.length === 1 && typeof error === 'string') {
  32. content = error
  33. error = null
  34. }
  35. if (error) {
  36. reject(error)
  37. } else {
  38. resolve(content)
  39. }
  40. }
  41. })
  42. return (process(content, file, done) || Promise.resolve()).then((content) => {
  43. if (content) {
  44. // async process correctly returned content
  45. return content
  46. }
  47. // process called done() (Either old sync api or an async function that did not return content)
  48. return donePromise
  49. })
  50. }
  51. async function runProcessors (preprocessors, file, content) {
  52. try {
  53. for (const process of preprocessors) {
  54. content = await executeProcessor(process, file, content)
  55. }
  56. } catch (error) {
  57. file.contentPath = null
  58. file.content = null
  59. throw error
  60. }
  61. file.contentPath = null
  62. file.content = content
  63. file.sha = CryptoUtils.sha1(content)
  64. }
  65. function createPriorityPreprocessor (config = {}, preprocessorPriority, basePath, instantiatePlugin) {
  66. _.union.apply(_, Object.values(config)).forEach((name) => instantiatePlugin('preprocessor', name))
  67. return async function preprocess (file) {
  68. const buffer = await tryToRead(file.originalPath, log)
  69. let isBinary = file.isBinary
  70. if (isBinary == null) {
  71. // Pattern did not specify, probe for it.
  72. isBinary = await isBinaryFile(buffer, buffer.length)
  73. }
  74. const preprocessorNames = Object.keys(config).reduce((ppNames, pattern) => {
  75. if (mm(file.originalPath, pattern, { dot: true })) {
  76. ppNames = _.union(ppNames, config[pattern])
  77. }
  78. return ppNames
  79. }, [])
  80. // Apply preprocessor priority.
  81. const preprocessors = preprocessorNames
  82. .map((name) => [name, preprocessorPriority[name] || 0])
  83. .sort((a, b) => b[1] - a[1])
  84. .map((duo) => duo[0])
  85. .reduce((preProcs, name) => {
  86. const p = instantiatePlugin('preprocessor', name)
  87. if (!isBinary || (p && p.handleBinaryFiles)) {
  88. preProcs.push(p)
  89. } else {
  90. log.warn(`Ignored preprocessing ${file.originalPath} because ${name} has handleBinaryFiles=false.`)
  91. }
  92. return preProcs
  93. }, [])
  94. await runProcessors(preprocessors, file, isBinary ? buffer : buffer.toString())
  95. }
  96. }
  97. createPriorityPreprocessor.$inject = ['config.preprocessors', 'config.preprocessor_priority', 'config.basePath', 'instantiatePlugin']
  98. exports.createPriorityPreprocessor = createPriorityPreprocessor