MikaAK/s3-plugin-webpack

View on GitHub
src/s3_plugin.js

Summary

Maintainability
C
1 day
Test Coverage
import http from 'http'
import https from 'https'
import fs from 'fs'
import path from 'path'
import ProgressBar from 'progress'
import cdnizer from 'cdnizer'
import _ from 'lodash'
import mime from 'mime/lite'
import {S3, CloudFront} from 'aws-sdk'

import packageJson from '../package.json'

import {
  addSeperatorToPath,
  addTrailingS3Sep,
  getDirectoryFilesRecursive,
  testRule,
  UPLOAD_IGNORES,
  DEFAULT_UPLOAD_OPTIONS,
  REQUIRED_S3_UP_OPTS,
  PATH_SEP,
  DEFAULT_TRANSFORM,
} from './helpers'

http.globalAgent.maxSockets = https.globalAgent.maxSockets = 50

const compileError = function(compilation, error) {
  compilation.errors.push(new Error(error))
}

module.exports = class S3Plugin {
  constructor(options = {}) {
    var {
      include,
      exclude,
      progress,
      basePath,
      directory,
      htmlFiles,
      basePathTransform = DEFAULT_TRANSFORM,
      s3Options = {},
      cdnizerOptions = {},
      s3UploadOptions = {},
      cloudfrontInvalidateOptions = {},
      priority
    } = options

    this.uploadOptions = s3UploadOptions
    this.cloudfrontInvalidateOptions = cloudfrontInvalidateOptions
    this.isConnected = false
    this.cdnizerOptions = cdnizerOptions
    this.urlMappings = []
    this.uploadTotal = 0
    this.uploadProgress = 0
    this.basePathTransform = basePathTransform
    basePath = basePath ? addTrailingS3Sep(basePath) : ''

    this.options = {
      directory,
      include,
      exclude,
      basePath,
      priority,
      htmlFiles: typeof htmlFiles === 'string' ? [htmlFiles] : htmlFiles,
      progress: _.isBoolean(progress) ? progress : true
    }

    this.clientConfig = {
      s3Options,
      maxAsyncS3: 50
    }

    this.noCdnizer = !Object.keys(this.cdnizerOptions).length

    if (!this.noCdnizer && !this.cdnizerOptions.files)
      this.cdnizerOptions.files = []
  }

  apply(compiler) {
    this.connect()

    const isDirectoryUpload = !!this.options.directory,
          hasRequiredUploadOpts = _.every(REQUIRED_S3_UP_OPTS, type => this.uploadOptions[type])

    // Set directory to output dir or custom
    this.options.directory = this.options.directory          ||
                             compiler.options.output.path    ||
                             compiler.options.output.context ||
                             '.'

    compiler.hooks.afterEmit.tapPromise(packageJson.name, async(compilation) => {
      var error

      if (!hasRequiredUploadOpts)
        error = `S3Plugin-RequiredS3UploadOpts: ${REQUIRED_S3_UP_OPTS.join(', ')}`

      if (error) return compileError(compilation, error)

      if (isDirectoryUpload) {
        const dPath = addSeperatorToPath(this.options.directory)

        return this.getAllFilesRecursive(dPath)
          .then((files) => this.handleFiles(files))
          .catch(e => this.handleErrors(e, compilation))
      } else {
        return this.getAssetFiles(compilation)
          .then((files) => this.handleFiles(files))
          .catch(e =>  this.handleErrors(e, compilation))
      }
    })
  }

  handleFiles(files) {
    return this.changeUrls(files)
      .then((files) => this.filterAllowedFiles(files))
      .then((files) => this.uploadFiles(files))
      .then(() => this.invalidateCloudfront())
  }

  async handleErrors(error, compilation) {
    compileError(compilation, `S3Plugin: ${error}`)
    throw error
  }

  getAllFilesRecursive(fPath) {
    return getDirectoryFilesRecursive(fPath)
  }

  addPathToFiles(files, fPath) {
    return files.map(file => ({name: file, path: path.resolve(fPath, file)}))
  }

  getFileName(file = '') {
    if (_.includes(file, PATH_SEP))
      return file.substring(_.lastIndexOf(file, PATH_SEP) + 1)
    else
      return file
  }

  getAssetFiles({assets}) {
    const files = _.map(assets, (value, name) => ({name, path: value.existsAt}))

    return Promise.resolve(files)
  }

  cdnizeHtml(file) {
    return new Promise((resolve, reject) => {
      fs.readFile(file.path, (err, data) => {
        if (err)
          return reject(err)

        fs.writeFile(file.path, this.cdnizer(data.toString()), (err) => {
          if (err)
            return reject(err)

          resolve(file)
        })
      })
    })
  }

  changeUrls(files = []) {
    if (this.noCdnizer)
      return Promise.resolve(files)

    var allHtml

    const {directory, htmlFiles = []} = this.options

    if (htmlFiles.length)
      allHtml = this.addPathToFiles(htmlFiles, directory).concat(files)
    else
      allHtml = files

    this.cdnizerOptions.files = allHtml.map(({name}) => `{/,}*${name}*`)
    this.cdnizer = cdnizer(this.cdnizerOptions)

    const [cdnizeFiles, otherFiles] = _(allHtml)
      .uniq('name')
      .partition((file) => /\.(html|css)/.test(file.name))
      .value()

    return Promise.all(cdnizeFiles.map(file => this.cdnizeHtml(file)).concat(otherFiles))
  }

  filterAllowedFiles(files) {
    return files.reduce((res, file) => {
      if (this.isIncludeAndNotExclude(file.name) && !this.isIgnoredFile(file.name))
        res.push(file)

      return res
    }, [])
  }

  isIgnoredFile(file) {
    return _.some(UPLOAD_IGNORES, ignore => new RegExp(ignore).test(file))
  }

  isIncludeAndNotExclude(file) {
    var isExclude,
        isInclude,
        {include, exclude} = this.options

    isInclude = include ? testRule(include, file) : true
    isExclude = exclude ? testRule(exclude, file) : false

    return isInclude && !isExclude
  }

  connect() {
    if (this.isConnected)
      return

    this.client = new S3(this.clientConfig.s3Options)
    this.isConnected = true
  }

  transformBasePath() {
    return Promise.resolve(this.basePathTransform(this.options.basePath))
      .then(addTrailingS3Sep)
      .then(nPath => this.options.basePath = nPath)
  }

  setupProgressBar(uploadFiles) {
    const progressTotal = uploadFiles
      .reduce((acc, {upload}) => upload.totalBytes + acc, 0)

    const progressBar = new ProgressBar('Uploading [:bar] :percent :etas', {
      complete: '>',
      incomplete: '∆',
      total: progressTotal
    })

    var progressValue = 0

    uploadFiles.forEach(function({upload}) {
      upload.on('httpUploadProgress', function({loaded}) {
        progressValue += loaded

        progressBar.update(progressValue)
      })
    })
  }

  prioritizeFiles(files) {
    const remainingFiles = [...files]
    const prioritizedFiles = this.options.priority
      .map(reg => _.remove(remainingFiles, (file) => reg.test(file.name)))


    return [remainingFiles, ...prioritizedFiles]
  }

  uploadPriorityChunk(priorityChunk) {
    const uploadFiles = priorityChunk.map(file => this.uploadFile(file.name, file.path))


    return Promise.all(uploadFiles.map(({promise}) => promise))
  }

  uploadInPriorityOrder(files) {
    const priorityChunks = this.prioritizeFiles(files)
    const uploadFunctions = priorityChunks
      .map(priorityChunk =>
        () => this.uploadPriorityChunk(priorityChunk))


    return uploadFunctions.reduce((promise, uploadFn) => promise.then(uploadFn), Promise.resolve())
  }

  uploadFiles(files = []) {
    return this.transformBasePath()
      .then(() => {
        if (this.options.priority) {
          return this.uploadInPriorityOrder(files)
        } else {
          const uploadFiles = files.map(file => this.uploadFile(file.name, file.path))

          if (this.options.progress) {
            this.setupProgressBar(uploadFiles)
          }

          return Promise.all(uploadFiles.map(({promise}) => promise))
        }
      })
  }

  uploadFile(fileName, file) {
    let Key = this.options.basePath + fileName
    const s3Params = _.mapValues(this.uploadOptions, (optionConfig) => {
      return _.isFunction(optionConfig) ? optionConfig(fileName, file) : optionConfig
    })

    // avoid noname folders in bucket
    if (Key[0] === '/')
      Key = Key.substr(1)

    if (s3Params.ContentType === undefined)
      s3Params.ContentType = mime.getType(fileName)

    const Body = fs.createReadStream(file)
    const upload = this.client.upload(
      _.merge({Key, Body}, DEFAULT_UPLOAD_OPTIONS, s3Params)
    )

    if (!this.noCdnizer)
      this.cdnizerOptions.files.push(`*${fileName}*`)

    return {upload, promise: upload.promise()}
  }

  invalidateCloudfront() {
    const {clientConfig, cloudfrontInvalidateOptions} = this

    if (cloudfrontInvalidateOptions.DistributionId) {
      const {accessKeyId, secretAccessKey, sessionToken} = clientConfig.s3Options
      const cloudfront = new CloudFront({accessKeyId, secretAccessKey, sessionToken})

      if (!_.isArray(cloudfrontInvalidateOptions.DistributionId))
        cloudfrontInvalidateOptions.DistributionId = [cloudfrontInvalidateOptions.DistributionId]

      const cloudfrontInvalidations = cloudfrontInvalidateOptions.DistributionId
        .map((DistributionId) => new Promise((resolve, reject) => {
          cloudfront.createInvalidation({
            DistributionId,
            InvalidationBatch: {
              CallerReference: Date.now().toString(),
              Paths: {
                Quantity: cloudfrontInvalidateOptions.Items.length,
                Items: cloudfrontInvalidateOptions.Items
              }
            }
          }, (err, res) => {
            if (err)
              reject(err)
            else
              resolve(res.Id)
          })
        }))

      return Promise.all(cloudfrontInvalidations)
    } else {
      return Promise.resolve(null)
    }
  }
}