ruby-analysis/delfos

#!/usr/bin/env ruby
# frozen_string_literal: true

require "fileutils"

# Credit to http://ruby-journal.com/how-to-import-millions-records-via-activerecord-within-minutes-not-hours/
# for the basis for this script

NUMBER_OF_SPLIT_LINES = 10_000
SPLIT_FILE_PREFIX     = "delfos_queries_"
TEMP_DIRECTORY        = "./tmp/delfos"
BIG_FILE_PATH         = ARGV[0] || "./tmp/delfos/query_parameters.json"

BIG_FILENAME = File.basename BIG_FILE_PATH
FileUtils.rm_rf TEMP_DIRECTORY
FileUtils.mkdir_p TEMP_DIRECTORY
FileUtils.cp BIG_FILE_PATH, "#{TEMP_DIRECTORY}/#{BIG_FILENAME}"

FileUtils.cd TEMP_DIRECTORY

def split_file
  puts "Split '#{BIG_FILENAME}' file into small chunks with size #{NUMBER_OF_SPLIT_LINES} lines"
  system "split -l #{NUMBER_OF_SPLIT_LINES} #{BIG_FILENAME} #{SPLIT_FILE_PREFIX}"
end

def process_split_files
  split_files = Dir.glob("#{SPLIT_FILE_PREFIX}*")

  split_files.each do |f|
    command = "FILE=#{f} delfos_import_offline_queries"
    system command
  end
end

split_file
process_split_files

system "delfos_update_distance"