lib/d_heap/benchmarks/rspec_matchers.rb
# frozen_string_literal: true
require "d_heap/benchmarks"
module DHeap::Benchmarks
# Profiles different implementations with different sizes
module RSpecMatchers # rubocop:disable Metrics/ModuleLength
extend RSpec::Matchers::DSL
# Assert ips (iterations per second):
#
# expect { ... }.to perform_at_least(1_000_000).ips
# .running_at_least(10).times # optional, defaults to 1
# .running_at_least(10).seconds # optional, defaults to 1s
# .running_at_most(10_000_000).times # optional, defaults to nil
# .running_at_most(2).seconds # optional, defaults to 2s
# .warmup_at_most(1000).times # optional, defaults to 1k
# .warmup_at_most(0.100).seconds # optional, defaults to 0.1s
# .iterations_per_round # optional, defaults to 1
# .and_at_least(1.1).times.faster_than { ... } # can also compare
#
# Assert comparison (and optionally runtime or ips):
#
# expect { ... }.to perform_at_least(2.5).times_faster_than { ... }
# .running_at_least(10).times # optional, defaults to 1
# .running_at_least(10).seconds # optional, defaults to 1s
# .running_at_most(10_000_000).times # optional, defaults to nil
# .running_at_most(2).seconds # optional, defaults to 2s
# .warmup_at_most(1000).times # optional, defaults to 1k
# .warmup_at_most(0.100).seconds # optional, defaults to 0.1s
# .iterations_per_call # optional, defaults to 1
# .and_at_least(100).ips { ... } # can also assert ips
#
# n.b: Given a known constant number of iterations, run time and ips are both
# measuring the same underlying metric.
#
# rubocop:disable Metrics/BlockLength, Layout/SpaceAroundOperators
matcher :perform_at_least do |expected|
supports_block_expectations
%i[
is_at_least
running_at_most
running_at_least
warmup_at_most
].each do |type|
chain type do |number|
reason, value = ___number_reason_and_value___
if reason || value
raise "Need to handle unit-less number first: %s(%p)" % [reason, value]
end
@number_for = type
@number_val = number
end
end
alias_method :and_at_least, :is_at_least
%i[
times
seconds
milliseconds
].each do |unit|
chain unit do
reason, value = ___number_reason_and_value___
raise "No number was specified" unless reason && value
apply_number_to_reason(reason, value, unit)
@number_for = @number_val = nil
end
end
# TODO: let IPS set time to run instead of iterations to run
chain :ips do
reason, value = ___number_reason_and_value___
raise "'ips' unit is only for assertions" unless reason == :is_at_least
raise "Already asserting %s ips" % [@expect_ips] if @expect_ips
raise "'ips' assertion has already been made" if @expect_ips
raise "Unknown assertion count" unless value
@expect_ips = Integer(value)
@number_for = @number_val = nil
end
# need to use method because "chain" can't take a block
def times_faster_than(&other)
reason, value = ___number_reason_and_value___
raise "'times_faster_than' is only for assertions" unless reason == :is_at_least
raise "Already asserting %sx comparison" % [@expect_cmp] if @expect_cmp
raise ArgumentError, "must provide a proc" unless other
@expect_cmp = Float(value)
@cmp_proc = other
@number_for = @number_val = nil
self
end
chain :loudly do @volume = :loud end
chain :quietly do @volume = :quiet end
chain :volume do |volume|
raise "Invalid volume" unless %i[loud quiet].include?(volume)
@volume = volume
end
chain :iterations_per_round do |iterations|
if @iterations_per_round
raise "Already set iterations per round (%p)" [@iterations_per_round]
end
@iterations_per_round = Integer(iterations)
end
match do |actual|
require "benchmark"
raise "Need to expect a proc or block" unless actual.respond_to?(:to_proc)
raise "Need a performance assertion" unless assertion?
@actual_proc = actual
prepare_for_measurement
if @max_iter && (@max_iter % @iterations_per_round) != 0
raise "Iterations per round (%p) must divide evenly by max iterations (%p)" % [
@iterations_per_round, @max_iter,
]
end
run_measurements
cmp_okay? && ips_okay?
end
description do
[
@expect_cmp && cmp_okay_msg,
@expect_ips && ips_okay_msg,
].join(", and ")
end
failure_message do
[
cmp_okay? ? nil : "expected to #{cmp_okay_msg} but #{cmp_fail_msg}", # =>
ips_okay? ? nil : "expected to #{ips_okay_msg} but #{ips_fail_msg}",
].compact.join(", and ")
end
private
chain :__convert_expected_to_ivars__ do
@number_val ||= expected
@number_for ||= :is_at_least if @number_val
expected = nil
end
private :__convert_expected_to_ivars__
def ___number_reason_and_value___
__convert_expected_to_ivars__
[@number_for, @number_val]
end
def apply_number_to_reason(reason, value, unit)
normalized_value, normalized_unit = normalize_unit(unit)
case reason
when :running_at_most; apply_max_run normalized_value, normalized_unit
when :running_at_least; apply_min_run normalized_value, normalized_unit
when :warmup_at_most; apply_warmup normalized_value, normalized_unit
else raise "%s is incompatible with %s(%p)" % [unit, reason, value]
end
end
def normalize_unit(unit)
case unit
when :seconds; [Float(@number_val), :seconds]
when :milliseconds; [Float(@number_val) / 1000.0, :seconds]
when :times; [Integer(@number_val), :times]
else raise "Invalid unit %s for %s(%p)" % [unit, reason, value]
end
end
def apply_min_run(value, unit)
case unit
when :seconds; @min_time = value
when :times; @min_iter = value
end
end
def apply_max_run(value, unit)
case unit
when :seconds; @max_time = value
when :times; @max_iter = value
end
end
def apply_warmup(value, unit)
case unit
when :seconds; @warmup_time = value
when :times; @warmup_iter = value
end
end
def prepare_for_measurement
@volume ||= ENV.fetch("RSPEC_BENCHMARK_VOLUME", :quiet).to_sym
@max_time ||= 2
@min_time ||= 1
@min_iter ||= 1
@warmup_time ||= 0.100
@warmup_iter ||= 1000
@iterations_per_round ||= 1
nil
end
def run_measurements
puts header if loud?
warmup
take_measurements
end
def header
max_rounds = @max_iter && @max_iter / @iterations_per_round
[
"Warmup time %s, or iterations: %s" % [@min_iter, @max_iter],
"Benchmark time (%s..%s) or iterations (%s..%s), max rounds: %p" % [
@min_time, @max_time, @min_iter, @max_iter, max_rounds,
],
"%-10s %s" % ["", Benchmark::CAPTION],
].join("\n")
end
def warmup
return unless 0 < @warmup_time && 0 < @warmup_iter # rubocop:disable Style/NumericPredicate
args = [@warmup_iter, 0, @warmup_time, 1, @warmup_iter]
measure("warmup", *args, &@actual_proc)
measure("warmup cmp", *args, &@cmp_proc) if @cmp_proc
end
def take_measurements
args = [@iterations_per_round, @min_time, @max_time, @min_iter, @max_iter]
@actual_tms = measure("actual", *args, &@actual_proc)
@cmp_tms = measure("cmp", *args, &@cmp_proc) if @cmp_proc
return unless @cmp_proc
# how many times faster?
@actual_cmp = @actual_tms.ips_real / @cmp_tms.ips_real
puts "Ran %0.3fx as fast as comparison" % [@actual_cmp] if loud?
end
def loud?; @volume == :loud end
def assertion?; !!(@expect_cmp || @expect_ips) end
def cmp_okay?; !@expect_cmp || @expect_cmp < @actual_cmp end
def ips_okay?; !@expect_tms || @expect_tms.ips < @actual_tms.ips end
def measure(name, ipr, *args)
measurements = TmsMeasurements.new(name, ipr, *args)
measurements.max_rounds.times do
# GC.start(full_mark: true, immediate_sweep: true)
# GC.compact
measurements << Benchmark.measure do
yield ipr
end
# p measurements.real
break if measurements.max_time < measurements.real
end
log_measurement(name, measurements)
measurements
end
# rubocop:disable Metrics/AbcSize
def units_str(num)
if num >= 10**12; "%7.3fT" % [num.to_f / 10**12]
elsif num >= 10** 9; "%7.3fB" % [num.to_f / 10** 9]
elsif num >= 10** 6; "%7.3fM" % [num.to_f / 10** 6]
elsif num >= 10** 3; "%7.3fk" % [num.to_f / 10** 3]
else "%7.3f" % [num.to_f]
end
end
# rubocop:enable Metrics/AbcSize
def log_measurement(name, measurements)
return unless loud?
puts "%-10s %s => %s ips (%d rounds)" % [
name,
measurements.tms.to_s.rstrip,
units_str(measurements.ips_real),
measurements.size,
]
end
def cmp_okay_msg; "run %0.2fx faster" % [@expect_cmp] end
def cmp_fail_msg; "was only %0.2fx as fast" % [@actual_cmp] end
def ips_okay_msg; "run with %s ips" % [units_str(@expect_ips)] end
def ips_fail_msg; "was only %s ips" % [units_str(@actual_ips)] end
end
# rubocop:enable Metrics/BlockLength, Layout/SpaceAroundOperators
alias_matcher :perform_with, :perform
end
# Replicates a subset of the functionality in benchmark-ips
#
# TODO: merge this with benchmark-ips
# TODO: implement (or remove) min_time, min_iter
class TmsMeasurements
attr_reader :iterations_per_entry
attr_reader :iterations
attr_reader :min_time
attr_reader :max_time
attr_reader :min_iter
attr_reader :max_iter
def initialize(name, ipe, min_time, max_time, min_iter, max_iter) # rubocop:disable Metrics/ParameterLists
@name = name
@iterations_per_entry = Integer(ipe)
@min_time = Float(min_time)
@max_time = Float(max_time)
@min_iter = Integer(min_iter)
@max_iter = Integer(max_iter)
@entries = []
@sum = Benchmark::Tms.new
@iterations = 0
end
def size; entries.size end
def <<(tms)
raise TypeError, "not a #{Benchmark::Tms}" unless tms.is_a?(Benchmark::Tms)
raise IndexError, "full" if @max_iter <= size
@sum += tms
@iterations += @iterations_per_entry
@entries << tms
self
end
def sum; @sum.dup end
alias tms sum
def entries; @entries.dup end
def cstime; @sum.cstime end
def cutime; @sum.cutime end
def real; @sum.real end
def stime; @sum.stime end
def total; @sum.total end
def utime; @sum.utime end
def ips_real; @iterations / real end
def ips_total; @iterations / total end
def ips_utime; @iterations / utime end
def max_rounds
@max_iter && @max_iter / @iterations_per_entry
end
end
end