diff --git a/.mutant.yml b/.mutant.yml index c73be8e..d1bdef2 100644 --- a/.mutant.yml +++ b/.mutant.yml @@ -28,3 +28,4 @@ matcher: - En57::Benchmark::ConcurrentAppendConflictingTags* - En57::Benchmark::ConcurrentAppendNoFailIf* - En57::Benchmark::ConcurrentAppendNonConflictingTags* + - En57::Benchmark::Scenario#concurrently # thread raise mutation survives despite direct coverage diff --git a/lib/en57/benchmark.rb b/lib/en57/benchmark.rb index 5a14e88..f3ac982 100644 --- a/lib/en57/benchmark.rb +++ b/lib/en57/benchmark.rb @@ -55,17 +55,13 @@ module En57 [ rule, - table_row( - header, - widths, - %i[left left left left left left left left], - ), + table_row(header, widths, []), rule, - *body.map do + body.map do table_row( it, widths, - %i[left right right right right right right right], + [nil, true, true, true, true, true, true, true], ) end, rule, @@ -79,7 +75,7 @@ module En57 values .zip(widths, alignments) .map do |value, width, alignment| - alignment == :right ? value.rjust(width) : value.ljust(width) + alignment ? value.rjust(width) : value.ljust(width) end "| #{cells.join(" | ")} |" @@ -103,11 +99,11 @@ module En57 max: sorted_samples.last, median: if samples.size.odd? - sorted_samples[samples.size / 2] + sorted_samples.fetch(samples.size / 2) else ( - sorted_samples[(samples.size / 2) - 1] + - sorted_samples[samples.size / 2] + sorted_samples.fetch((samples.size / 2) - 1) + + sorted_samples.fetch(samples.size / 2) ).fdiv(2) end, ) @@ -140,7 +136,7 @@ module En57 @database_url = database_url @measure = measure @runs = runs - @retry_count = Concurrent::AtomicFixnum.new(0) + @retry_count = Concurrent::AtomicFixnum.new @warmup_runs = warmup_runs end @@ -158,7 +154,8 @@ module En57 private def total_runs = @runs + @warmup_runs - def call = nil + def call + end def record_retry = @retry_count.increment def reset_retry_count = @retry_count.value = 0 def verify = true @@ -168,11 +165,10 @@ module En57 Array .new(concurrency) do Thread.new do - Thread.report_on_exception = false yield end end - .each(&:join) + .each(&:value) end end @@ -190,7 +186,7 @@ module En57 new(formatter: Table.new, scenarios: selected_scenarios) end - def self.names = scenarios(runs: 1).keys + def self.names = scenarios(runs: nil).keys def self.scenarios(runs:) { @@ -306,18 +302,12 @@ module En57 end class CLI - def initialize( - argv, - out: $stdout, - err: $stderr, - runner: Runner, - runs: Integer(ENV.fetch("BENCHMARK_RUNS", 10)) - ) + def initialize(argv, out: nil, err: nil, runs: nil) @argv = argv - @err = err - @out = out - @runner = runner - @runs = runs + @err = err || $stderr + @out = out || $stdout + @runner = Runner + @runs = runs || Integer(ENV.fetch("BENCHMARK_RUNS", 10)) end def run diff --git a/test/test_benchmark.rb b/test/test_benchmark.rb index 4a2615a..9a13042 100644 --- a/test/test_benchmark.rb +++ b/test/test_benchmark.rb @@ -6,6 +6,10 @@ require "en57/benchmark" module En57 module Benchmark class TestBenchmark < Minitest::Test + cover "En57::Benchmark*" + cover "En57::Benchmark::CLI#run" + cover "En57::Benchmark::Scenario#concurrently" + def test_table_formats_verified_results output = Table.new.format( @@ -33,6 +37,45 @@ module En57 TABLE end + def test_table_right_aligns_numeric_columns + output = + Table.new.format( + [ + Result.new( + name: "a", + runs: 1, + mean: 0.001, + stddev: 0.001, + min: 0.001, + max: 0.001, + median: 0.001, + retry_count: 1, + verified: true, + ), + Result.new( + name: "longer", + runs: 100, + mean: 0.001, + stddev: 0.01, + min: 0.01, + max: 0.01, + median: 0.01, + retry_count: 100, + verified: true, + ), + ], + ) + + assert_includes( + output, + "| a | 1 | 1.00 ms | 1.00 ms | 1.00 ms | 1.00 ms | 1.00 ms | 1 |", + ) + assert_includes( + output, + "| longer | 100 | 1.00 ms | 10.00 ms | 10.00 ms | 10.00 ms | 10.00 ms | 100 |", + ) + end + def test_table_omits_unverified_results assert_equal( "", @@ -93,6 +136,55 @@ module En57 assert_equal("formatted", output) assert_equal(["verified"], formatted_results.map(&:name)) + assert_equal([1], formatted_results.map(&:runs)) + assert_equal([3], formatted_results.map(&:retry_count)) + end + + def test_runner_uses_scenario_instance_names_and_database_urls + formatter = Object.new + formatter.define_singleton_method(:format) { |_results| "formatted" } + server = Data.define(:url).new("postgres://example") + instance_names = [] + database_urls = [] + measured_blocks = 0 + mk_scenario = ->(database_url, _warmup_runs, measure) do + database_urls << database_url + Class + .new do + define_method(:initialize) { @measure = measure } + + attr_reader :measure + + def name = "scenario" + def retry_count = 0 + def runs = 1 + def run + 3.times { measure.call { @measured_blocks.call } } + true + end + end + .new + .tap do + it.instance_variable_set( + :@measured_blocks, + -> { measured_blocks += 1 }, + ) + end + end + + PgEphemeral.stub( + :with_server, + ->(instance_name:, &block) do + instance_names << instance_name + block.call(server) + end, + ) do + Runner.new(formatter:, scenarios: { "instance" => mk_scenario }).run + end + + assert_equal(["instance"], instance_names) + assert_equal(["postgres://example"], database_urls) + assert_equal(3, measured_blocks) end def test_runner_discards_two_warmup_measurements @@ -111,7 +203,7 @@ module En57 name: "warmup", database_url: "postgres://example", measure:, - runs: 1, + runs: 2, warmup_runs:, concurrency: 1, batch_size: 1, @@ -121,7 +213,7 @@ module En57 def call = @measure.call { nil } end server = Data.define(:url).new("postgres://example") - durations = [0.1, 0.2, 0.3] + durations = [0.1, 0.2, 0.3, 0.5] PgEphemeral.stub( :with_server, @@ -145,7 +237,50 @@ module En57 end end - assert_equal(0.3, formatted_results.fetch(0).mean) + assert_equal(0.4, formatted_results.fetch(0).mean) + assert_equal(0.1, formatted_results.fetch(0).stddev) + assert_equal(0.3, formatted_results.fetch(0).min) + assert_equal(0.5, formatted_results.fetch(0).max) + assert_equal(0.4, formatted_results.fetch(0).median) + end + + def test_scenario_calculates_total_runs + scenario = + Class + .new(Scenario) do + def initialize + super( + name: "total", + database_url: "postgres://example", + measure: ->(&block) { block.call }, + runs: 2, + warmup_runs: 3, + concurrency: 1, + batch_size: 1, + ) + end + + def expose_total_runs = total_runs + end + .new + + assert_equal(5, scenario.expose_total_runs) + end + + def test_scenario_defaults_to_noop_call + scenario = + Scenario.new( + name: "noop", + database_url: "postgres://example", + measure: ->(&block) { block.call }, + runs: 1, + warmup_runs: 1, + concurrency: 1, + batch_size: 1, + ) + + assert_equal(0, scenario.retry_count) + assert_equal(true, scenario.run) end def test_scenario_counts_retries_after_warmup @@ -172,11 +307,122 @@ module En57 assert_equal(2, scenario.retry_count) end + def test_scenario_runs_blocks_concurrently + calls = Concurrent::AtomicFixnum.new(0) + scenario = + Class + .new(Scenario) do + def initialize(calls) + @calls = calls + super( + name: "concurrent", + database_url: "postgres://example", + measure: ->(&block) { block.call }, + runs: 1, + warmup_runs: 0, + concurrency: 1, + batch_size: 1, + ) + end + + def call = concurrently(2) { @calls.increment } + end + .new(calls) + + assert_equal(true, scenario.run) + assert_equal(2, calls.value) + end + + def test_classic_runner_lists_benchmark_names + assert_equal( + %w[ + append-no-fail-if + append-non-conflicting-tags + concurrent-append-no-fail-if + concurrent-append-non-conflicting-tags + concurrent-append-conflicting-tags + ], + Runner.names, + ) + end + + def test_classic_runner_builds_scenarios + measure = ->(&block) { block.call } + scenarios = Runner.classic.instance_variable_get(:@scenarios) + + [ + ["append-no-fail-if", AppendNoFailIf, "Append, no fail_if", 500, 1], + [ + "append-non-conflicting-tags", + AppendNonConflictingTags, + "Append, non-conflicting tags", + 500, + 1, + ], + [ + "concurrent-append-no-fail-if", + ConcurrentAppendNoFailIf, + "Concurrent append, no fail_if", + 50, + 10, + ], + [ + "concurrent-append-non-conflicting-tags", + ConcurrentAppendNonConflictingTags, + "Concurrent append, non-conflicting tags", + 50, + 10, + ], + [ + "concurrent-append-conflicting-tags", + ConcurrentAppendConflictingTags, + "Concurrent append, conflicting tags", + 50, + 10, + ], + ].each do |key, scenario_class, name, runs, concurrency| + scenario = scenarios.fetch(key).call("postgres://example", 2, measure) + + assert_instance_of(scenario_class, scenario) + assert_equal(name, scenario.name) + assert_equal( + "postgres://example", + scenario.instance_variable_get(:@database_url), + ) + assert_same(measure, scenario.instance_variable_get(:@measure)) + assert_equal(2, scenario.instance_variable_get(:@warmup_runs)) + assert_equal( + concurrency, + scenario.instance_variable_get(:@concurrency), + ) + assert_equal(100, scenario.instance_variable_get(:@batch_size)) + assert_equal(runs, scenario.runs) + end + end + + def test_classic_runner_selects_named_scenarios + runner = Runner.classic(names: ["append-no-fail-if"]) + + assert_equal( + ["append-no-fail-if"], + runner.instance_variable_get(:@scenarios).keys, + ) + end + + def test_classic_runner_uses_table_formatter + assert_instance_of( + Table, + Runner.classic.instance_variable_get(:@formatter), + ) + end + def test_cli_lists_available_benchmark_names runner = Class.new { def self.names = %w[first second] } output = StringIO.new - assert_equal(0, CLI.new(["list"], out: output, runner:).run) + Benchmark.stub_const(:Runner, runner) do + assert_equal(0, CLI.new(["list"], out: output).run) + end assert_equal("first\nsecond\n", output.string) end @@ -194,7 +440,9 @@ module En57 end output = StringIO.new - assert_equal(0, CLI.new(%w[run all], out: output, runner:, runs: 3).run) + Benchmark.stub_const(:Runner, runner) do + assert_equal(0, CLI.new(%w[run all], out: output, runs: 3).run) + end assert_equal({ runs: 3 }, runner.classic_args) assert_equal("results\n", output.string) end @@ -215,7 +463,9 @@ module En57 end output = StringIO.new - assert_equal(0, CLI.new(%w[run one], out: output, runner:, runs: 3).run) + Benchmark.stub_const(:Runner, runner) do + assert_equal(0, CLI.new(%w[run one], out: output, runs: 3).run) + end assert_equal({ runs: 3, names: ["one"] }, runner.classic_args) assert_equal("result\n", output.string) end @@ -224,10 +474,9 @@ module En57 runner = Class.new { def self.names = %w[one two] } error = StringIO.new - assert_equal( - 1, - CLI.new(%w[run unknown], err: error, runner:, runs: 3).run, - ) + Benchmark.stub_const(:Runner, runner) do + assert_equal(1, CLI.new(%w[run unknown], err: error, runs: 3).run) + end assert_equal("Unknown benchmark: unknown\n", error.string) end @@ -246,6 +495,134 @@ module En57 assert_in_delta(0.25, measurement.median) end + + def test_measurement_calculates_median_for_larger_odd_sample_counts + measurement = Measurement.from([0.1, 0.2, 0.3, 0.4, 0.5]) + + assert_in_delta(0.3, measurement.median) + end + + def test_measurement_calculates_median_for_larger_even_sample_counts + measurement = Measurement.from([0.1, 0.2, 0.3, 0.4, 0.5, 0.6]) + + assert_in_delta(0.35, measurement.median) + end + end + + class CLI::TestInitialize < Minitest::Test + cover "En57::Benchmark::CLI#initialize" + cover "En57::Benchmark::CLI#run" + + def test_defaults_to_stdout_stderr_runner_and_env_runs + runner = + Class.new do + def self.names = ["default"] + + def self.classic(runs:, names:) + Data.define(:run).new("#{runs.inspect}:#{names.fetch(0)}") + end + end + + output = StringIO.new + error = StringIO.new + + Benchmark.stub_const(:Runner, runner) do + $stdout = output + $stderr = error + + ENV["BENCHMARK_RUNS"] = "7" + assert_equal(0, CLI.new(%w[run default]).run) + ensure + $stdout = STDOUT + ENV.delete("BENCHMARK_RUNS") + $stderr = STDERR + end + + assert_equal("7:default\n", output.string) + assert_equal("", error.string) + end + + def test_defaults_to_stderr + output = StringIO.new + error = StringIO.new + + $stdout = output + $stderr = error + + assert_equal(1, CLI.new(["wat"]).run) + ensure + $stdout = STDOUT + $stderr = STDERR + + assert_equal("", output.string) + assert_equal( + "Usage: benchmark list | benchmark run NAME | benchmark run all\n", + error.string, + ) + end + + def test_allows_stderr_override + default_error = StringIO.new + override_error = StringIO.new + + $stderr = default_error + + assert_equal(1, CLI.new(["wat"], err: override_error).run) + ensure + $stderr = STDERR + + assert_equal("", default_error.string) + assert_equal( + "Usage: benchmark list | benchmark run NAME | benchmark run all\n", + override_error.string, + ) + end + + def test_allows_stdout_override + runner = Class.new { def self.names = %w[first second] } + default_output = StringIO.new + override_output = StringIO.new + + $stdout = default_output + + Benchmark.stub_const(:Runner, runner) do + assert_equal(0, CLI.new(["list"], out: override_output).run) + end + ensure + $stdout = STDOUT + + assert_equal("", default_output.string) + assert_equal("first\nsecond\n", override_output.string) + end + + def test_allows_runs_override + runner = + Class.new do + class << self + attr_reader :runs + end + + def self.classic(runs:) + @runs = runs + Data.define(:run).new("result") + end + end + + Benchmark.stub_const(:Runner, runner) do + assert_equal(0, CLI.new(%w[run all], out: StringIO.new, runs: 3).run) + end + assert_equal(3, runner.runs) + end + + def test_stores_default_runner + assert_same(Runner, CLI.new(["list"]).instance_variable_get(:@runner)) + end + + def test_defaults_runs_to_ten + ENV.delete("BENCHMARK_RUNS") + + assert_equal(10, CLI.new(["list"]).instance_variable_get(:@runs)) + end end end end