bin/bench_regression
#!/usr/bin/env ruby
require 'fileutils'
require 'pathname'
require 'shellwords'
require 'English'
############################
# USAGE
#
# bundle exec bin/bench_regression <ref1> <ref2>
# <ref1> defaults to the current branch
# <ref2> defaults to the master branch
# bundle exec bin/bench_regression current # will run on the current branch
# bundle exec bin/bench_regression revisions 792fb8a90 master # every revision inclusive
# bundle exec bin/bench_regression 792fb8a90 master --repeat-count 2 --env CACHE_ON=off
# bundle exec bin/bench_regression vendor
###########################
class BenchRegression
ROOT = Pathname File.expand_path(File.join(*['..', '..']), __FILE__)
TMP_DIR_NAME = File.join('tmp', 'bench')
TMP_DIR = File.join(ROOT, TMP_DIR_NAME)
E_TMP_DIR = Shellwords.shellescape(TMP_DIR)
load ROOT.join('bin', 'bench')
attr_reader :source_stasher
def initialize
@source_stasher = SourceStasher.new
end
class SourceStasher
attr_reader :gem_require_paths, :gem_paths
attr_writer :vendor
def initialize
@gem_require_paths = []
@gem_paths = []
refresh_temp_dir
@vendor = false
end
def temp_dir_empty?
File.directory?(TMP_DIR) &&
Dir[File.join(TMP_DIR, '*')].none?
end
def empty_temp_dir
return if @vendor
return if temp_dir_empty?
FileUtils.mkdir_p(TMP_DIR)
Dir[File.join(TMP_DIR, '*')].each do |file|
if File.directory?(file)
FileUtils.rm_rf(file)
else
FileUtils.rm(file)
end
end
end
def fill_temp_dir
vendor_files(Dir[File.join(ROOT, 'test', 'benchmark', '*.{rb,ru}')])
# vendor_file(File.join('bin', 'bench'))
housekeeping { empty_temp_dir }
vendor_gem('benchmark-ips')
end
def vendor_files(files)
files.each do |file|
vendor_file(file)
end
end
def vendor_file(file)
FileUtils.cp(file, File.join(TMP_DIR, File.basename(file)))
end
def vendor_gem(gem_name)
directory_name = `bundle exec gem unpack benchmark-ips --target=#{E_TMP_DIR}`[/benchmark-ips.+\d/]
gem_paths << File.join(TMP_DIR, directory_name)
gem_require_paths << File.join(TMP_DIR_NAME, directory_name, 'lib')
housekeeping { remove_vendored_gems }
end
def remove_vendored_gems
return if @vendor
FileUtils.rm_rf(*gem_paths)
end
def refresh_temp_dir
empty_temp_dir
fill_temp_dir
end
def housekeeping
at_exit { yield }
end
end
module RevisionMethods
module_function
def current_branch
@current_branch ||= `cat .git/HEAD | cut -d/ -f3,4,5`.chomp
end
def current_revision
`git rev-parse --short HEAD`.chomp
end
def revision_description(rev)
`git log --oneline -1 #{rev}`.chomp
end
def revisions(start_ref, end_ref)
cmd = "git rev-list --reverse #{start_ref}..#{end_ref}"
`#{cmd}`.chomp.split("\n")
end
def checkout_ref(ref)
`git checkout #{ref}`.chomp
if $CHILD_STATUS
STDERR.puts "Checkout failed: #{ref}, #{$CHILD_STATUS.exitstatus}" unless $CHILD_STATUS.success?
$CHILD_STATUS.success?
else
true
end
end
def clean_head
system('git reset --hard --quiet')
end
end
module ShellMethods
def sh(cmd)
puts cmd
# system(cmd)
run(cmd)
# env = {}
# # out = STDOUT
# pid = spawn(env, cmd)
# Process.wait(pid)
# pid = fork do
# exec cmd
# end
# Process.waitpid2(pid)
# puts $CHILD_STATUS.exitstatus
end
require 'pty'
# should consider trapping SIGINT in here
def run(cmd)
puts cmd
child_process = ''
result = ''
# http://stackoverflow.com/a/1162850
# stream output of subprocess
begin
PTY.spawn(cmd) do |stdin, _stdout, pid|
begin
# Do stuff with the output here. Just printing to show it works
stdin.each do |line|
print line
result << line
end
child_process = PTY.check(pid)
rescue Errno::EIO
puts 'Errno:EIO error, but this probably just means ' \
'that the process has finished giving output'
end
end
rescue PTY::ChildExited
puts 'The child process exited!'
end
unless (child_process && child_process.success?)
exitstatus = child_process.exitstatus
puts "FAILED: #{child_process.pid} exited with status #{exitstatus.inspect} due to failed command #{cmd}"
exit exitstatus || 1
end
result
end
def bundle(ref)
system("rm -f Gemfile.lock")
# This is absolutely critical for bundling to work
Bundler.with_clean_env do
system("bundle check ||
bundle install --local ||
bundle install ||
bundle update")
end
# if $CHILD_STATUS
# STDERR.puts "Bundle failed at: #{ref}, #{$CHILD_STATUS.exitstatus}" unless $CHILD_STATUS.success?
# $CHILD_STATUS.success?
# else
# false
# end
end
end
include ShellMethods
include RevisionMethods
def benchmark_refs(ref1: nil, ref2: nil, cmd:)
checking_out = false
ref0 = current_branch
ref1 ||= current_branch
ref2 ||= 'master'
p [ref0, ref1, ref2, current_revision]
run_benchmark_at_ref(cmd, ref1)
p [ref0, ref1, ref2, current_revision]
run_benchmark_at_ref(cmd, ref2)
p [ref0, ref1, ref2, current_revision]
checking_out = true
checkout_ref(ref0)
rescue Exception # rubocop:disable Lint/RescueException
STDERR.puts "[ERROR] #{$!.message}"
checkout_ref(ref0) unless checking_out
raise
end
def benchmark_revisions(ref1: nil, ref2: nil, cmd:)
checking_out = false
ref0 = current_branch
ref1 ||= current_branch
ref2 ||= 'master'
revisions(ref1, ref2).each do |rev|
STDERR.puts "Checking out: #{revision_description(rev)}"
run_benchmark_at_ref(cmd, rev)
clean_head
end
checking_out = true
checkout_ref(ref0)
rescue Exception # rubocop:disable Lint/RescueException
STDERR.puts "[ERROR]: #{$!.message}"
checkout_ref(ref0) unless checking_out
raise
end
def run_benchmark_at_ref(cmd, ref)
checkout_ref(ref)
run_benchmark(cmd, ref)
end
def run_benchmark(cmd, ref = nil)
ref ||= current_revision
bundle(ref) &&
benchmark_tests(cmd, ref)
end
def benchmark_tests(cmd, ref)
base = E_TMP_DIR
# cmd.sub('bin/bench', 'tmp/revision_runner/bench')
# bundle = Gem.bin('bunle'
# Bundler.with_clean_env(&block)
# cmd = Shellwords.shelljoin(cmd)
# cmd = "COMMIT_HASH=#{ref} BASE=#{base} bundle exec ruby -rbenchmark/ips #{cmd}"
# Add vendoring benchmark/ips to load path
# CURRENT THINKING: IMPORTANT
# Pass into require statement as RUBYOPTS i.e. via env rather than command line argument
# otherwise, have a 'fast ams benchmarking' module that extends benchmarkings to add the 'ams'
# method but doesn't depend on benchmark-ips
options = {
commit_hash: ref,
base: base,
rubyopt: Shellwords.shellescape("-Ilib:#{source_stasher.gem_require_paths.join(':')}")
}
BenchmarkDriver.parse_argv_and_run(ARGV.dup, options)
end
end
if $PROGRAM_NAME == __FILE__
benchmarking = BenchRegression.new
case ARGV[0]
when 'current'
# Run current branch only
# super simple command line parsing
args = ARGV.dup
_ = args.shift # remove 'current' from args
cmd = args
benchmarking.run_benchmark(cmd)
when 'revisions'
# Runs on every revision
# super simple command line parsing
args = ARGV.dup
_ = args.shift
ref1 = args.shift # remove 'revisions' from args
ref2 = args.shift
cmd = args
benchmarking.benchmark_revisions(ref1: ref1, ref2: ref2, cmd: cmd)
when 'vendor'
# Just prevents vendored files from being cleaned up
# at exit. (They are vendored at initialize.)
benchmarking.source_stasher.vendor = true
else
# Default: Compare current_branch to master
# Optionally: pass in two refs as args to `bin/bench_regression`
# TODO: Consider checking across more revisions, to automatically find problems.
# super simple command line parsing
args = ARGV.dup
ref1 = args.shift
ref2 = args.shift
cmd = args
benchmarking.benchmark_refs(ref1: ref1, ref2: ref2, cmd: cmd)
end
end