npolar/argos-ruby

#!/usr/bin/env ruby
# encoding: utf-8

# argos-ascii /path/argos/ --dest=/path/dest --level=info 2>> /path/argos-json.log

Dir.chdir(__dir__) do
  require "bundler/setup"
  require_relative "../lib/argos"
end

require "optparse"
require "fileutils"
require "csv"

param = { format: "json",
  action: "parse",
  level: Logger::WARN,
  dest: nil,
  filter: nil,
}

optparser = OptionParser.new(ARGV) do |opts|
  #opts.banner ""

  opts.on_tail("--version", "-v", "Library version") do
    puts Argos.library_version
    exit
  end

  opts.on("--action={parse|source}", "-a", "Action") do |action|
    param[:action] = action
  end

  opts.on("--dest=destination", "-a", "Destination directory") do |dest|
    param[:dest] = dest
  end

  opts.on("--format=FORMAT", "-f", "Format") do |format|
    param[:format] = format
  end

  opts.on("--filter=FILTER", "Filter lambda") do |filter|
    param[:filter] = filter
  end

  opts.on("--level=LEVEL", "-l=LEVEL", "Log level: debug|info|warn|error|fatal") do |level|

    param[:level] = case level
      when /debug|0/i
        Logger::DEBUG
      when /info|1/i
        Logger::INFO
      when /warn|2/i
        Logger::WARN
      when /error|3/i
        Logger::ERROR
      when /fatal|4/i
        Logger::FATAL
      else
        param[:level]
    end
  end

   opts.on("--debug") do

    param[:level] = Logger::DEBUG
  end

end
optparser.parse!
glob = ARGV[0]||Dir.pwd

begin

log = Logger.new(STDERR)
log.level = param[:level]


if not Argos::Ascii.argos? glob
  glob = glob.gsub(/\/$/, "")
  if glob != /\*/
    glob += "/**/*"
  end
end

log.debug "#{File.realpath(__FILE__)} #{param}"
bundle = Digest::SHA1.hexdigest(glob+param[:filter].to_s)
result = []
argosfiles = 0

Dir[glob].reject {|f| File.directory? f}.select {|f|
  argosfiles = argosfiles + 1
  if not Argos::Ascii.argos? f
    log.warn "Not Argos: #{f}"
  end

  Argos::Ascii.argos? f
}.map {|filename|
  log.debug "*"*80
  argos = Argos::Ascii.factory(Argos::Ascii.type(filename))
  argos.filename = filename
  argos.filter = param[:filter]
  argos.bundle = bundle
  argos.log = log

  case param[:action]
    when "source","metadata"
      result << Argos::Ascii.source(argos).merge(glob: glob, bundle: bundle)

    when "parse"

      arr = argos.parse(filename)

      unless param[:dest].nil?
        jsonfile = "#{param[:dest].gsub(/\$/, "")}/#{argos.source}.json"
        File.open(jsonfile, "w") { |file| file.write(arr.to_json) }
      end
      result += arr
    when "messages"
      argos.parse(filename)
      result += argos.messages
  end
}

ds_count = result.select {|argos| argos[:type] == "ds"}.size
diag_count = result.select {|argos| argos[:type] == "diag"}.size
# err count
# warn count

if ["parse", "messages"].include? param[:action]
  log.info "Documents: #{result.size}, ds: #{ds_count}, diag: #{diag_count}, bundle: #{bundle}, glob: #{glob}"


elsif /source|metadata/ =~ param[:action]
  # Count the total number of files in the bundle
  result = result.select {|s| s[:size] > 0 }
  sum_count = result.map {|s| s[:size]}.inject { |sum, c| sum + c }
  result = result.map {|s|
    s[:total] = sum_count
    s
  }
  unless param[:dest].nil?
    jsonfile = "#{param[:dest].gsub(/\$/, "")}/bundle-#{bundle}-metadata.json"
    FileUtils.mkpath(File.dirname(jsonfile))
    log.debug "Argos bundle metadata: #{jsonfile}"
    File.open(jsonfile, "w") { |file| file.write(result.to_json) }

  end
  if argosfiles != result.size
    log.warn "Argos file count: #{argosfiles} differs from parsed files: #{result.size}"
  end
  log.info "Documents (Σcount): #{sum_count}, sources: #{result.size} (ds: #{ds_count}, diag: #{diag_count}), bundle: #{bundle}, glob: #{glob}"
end

if param[:dest].nil?
  case param[:format]
    when "json"
      puts JSON.pretty_generate(result)
    when /y(a)?ml/
      puts result.to_yaml
    when /(ruby|rb)/
      puts result
    when "csv"

      data = result.map {|r|
        if r.values.size != r.keys.size
          raise "Header/value size mismatch: #{r.values.size}/#{r.keys.size}"
        end
        r.values
      }
      csv = CSV.generate() do |csv|
        csv << result[0].keys
        data.each { |row| csv << row }
      end
      puts csv
  end
end

exit(0)

rescue => e
  Logger.new(STDERR).fatal e
  exit(1)
end