taskrabbit/elasticsearch-dump

#!/usr/bin/env node

const argv = require('minimist')(process.argv)
const path = require('path')
const fs = require('fs')
const Elasticdump = require(path.join(__dirname, '..', 'elasticdump.js'))
const packageData = require(path.join(__dirname, '..', 'package.json'))
const { isUrl } = require(path.join(__dirname, '..', 'lib', 'is-url.js'))
const ArgParser = require(path.join(__dirname, '..', 'lib', 'argv.js'))
const versionCheck = require(path.join(__dirname, '..', 'lib', 'version-check.js'))
require('aws-sdk/lib/maintenance_mode_message').suppress = true;


// For future developers.  If you add options here, be sure to add the option to test suite tests where necessary
const defaults = {
  size: -1,
  limit: 100,
  offset: 0,
  debug: false,
  type: 'data',
  delete: false,
  'delete-with-routing': false,
  'skip-existing': false,
  maxSockets: null,
  input: null,
  'input-index': null,
  output: null,
  'output-index': null,
  noRefresh: false,
  esCompress: false,
  inputTransport: null,
  outputTransport: null,
  searchBody: null,
  searchBodyTemplate: null,
  searchWithTemplate: false,
  filterSystemTemplates: true,
  templateRegex: '^(metrics|logs|.+_audit_log|.+-index-template|\\..+)$',
  headers: null,
  'input-headers': null,
  'output-headers': null,
  sourceOnly: false,
  jsonLines: false,
  format: '',
  'ignore-errors': false,
  'ignore-es-write-errors': true,
  'support-big-int': false,
  'big-int-fields': '',
  scrollId: null,
  scrollTime: '10m',
  scrollRetryDelay: 15000,
  'scroll-with-post': false,
  timeout: null,
  toLog: null,
  quiet: false,
  awsChain: false,
  awsAccessKeyId: null,
  awsSecretAccessKey: null,
  awsIniFileProfile: null,
  awsService: null,
  awsRegion: null,
  awsUrlRegex: null,
  s3AccessKeyId: null,
  s3SecretAccessKey: null,
  s3Region: null,
  s3Endpoint: null,
  s3SSLEnabled: true,
  s3ForcePathStyle: false,
  s3Compress: false,
  s3ServerSideEncryption: null,
  s3SSEKMSKeyId: null,
  s3ACL: null,
  s3StorageClass: 'STANDARD',
  s3Options: null,
  s3Configs: null,
  fsCompress: false,
  awsIniFileName: null,
  sessionToken: null,
  transform: null,
  httpAuthFile: null,
  params: null,
  'input-params': null,
  'output-params': null,
  prefix: '',
  suffix: '',
  retryAttempts: 0,
  customBackoff: false,
  retryDelayBase: 0,
  retryDelay: 5000,
  parseExtraFields: '',
  bulkAction: 'index',
  fileSize: -1,
  maxRows: -1,
  cert: null,
  key: null,
  pass: null,
  ca: null,
  tlsAuth: false,
  'input-cert': null,
  'input-key': null,
  'input-pass': null,
  'input-ca': null,
  'output-cert': null,
  'output-key': null,
  'output-pass': null,
  'output-ca': null,
  inputSocksProxy: null,
  inputSocksPort: null,
  outputSocksProxy: null,
  outputSocksPort: null,
  concurrency: 1,
  throttleInterval: 1,
  carryoverConcurrencyCount: true,
  intervalCap: 5,
  concurrencyInterval: 5000,
  overwrite: false,
  handleVersion: false,
  versionType: null,

  // csv options
  csvConfigs: null,
  csvDelimiter: ',',
  csvRowDelimiter: '\n',
  csvFirstRowAsHeaders: true,
  csvRenameHeaders: false,
  csvWriteHeaders: true,
  csvHandleNestedData: false,
  csvIdColumn: null,
  csvIndexColumn: null,
  csvTypeColumn: null,
  csvCustomHeaders: null,
  csvIgnoreEmpty: false,
  csvIgnoreAutoColumns: false,
  csvMaxRows: 0,
  csvSkipLines: 0,
  csvSkipRows: 0,
  csvTrim: false,
  csvRTrim: false,
  csvLTrim: false,
  csvDiscardUnmappedColumns: false,
  csvQuoteChar: '"',
  csvEscapeChar: '"',
  csvIncludeEndRowDelimiter: false,

  // opensearch compatability
  'force-os-version': '7.10.2'
}
const options = {}

versionCheck()

const args = new ArgParser({ options, parseJSONOpts: true })
args.parse(argv, defaults)

if (argv.debug === true) {
  process
    .on('unhandledRejection', (reason, p) => {
      console.error(reason, 'Unhandled Rejection at Promise', p)
    })
    .on('uncaughtException', err => {
      console.error(err, 'Uncaught Exception thrown')
      process.exit(1)
    })
}

if (argv.help === true) {
  let helpText = require('fs').readFileSync(path.join(__dirname, '..', 'lib', 'help.txt'))
  helpText = helpText.toString()
  helpText = helpText.replace('%%version%%', packageData.version)
  console.log(helpText)
} else if (argv.version === true) {
  console.log(packageData.version)
} else {
  if (!options.overwrite && options.output !== '$' && !isUrl(options.output) && fs.existsSync(options.output)) {
    console.error(`File \`${options.output}\` already exists, quitting`)
    process.exit(1)
  }

  const dumper = new Elasticdump(options)

  dumper.on('log', function (message) { args.log('log', message) })
  dumper.on('debug', function (message) { args.log('debug', message) })
  dumper.on('error', function (error) { args.log('error', `Error Emitted => ${error.message || JSON.stringify(error)}`) })

  dumper.dump(function (error) {
    if (error) {
      process.exit(1)
    }
  })
}