diff --git a/bin/benchmark b/bin/benchmark index b1265ca..2154cba 100755 --- a/bin/benchmark +++ b/bin/benchmark @@ -2,6 +2,4 @@ require_relative "../lib/en57/benchmark" -puts En57::Benchmark::Runner.classic( - runs: Integer(ENV.fetch("BENCHMARK_RUNS", 10)), - ).run +exit En57::Benchmark::CLI.new(ARGV).run diff --git a/lib/en57/benchmark.rb b/lib/en57/benchmark.rb index 2c3e19e..8ee78e2 100644 --- a/lib/en57/benchmark.rb +++ b/lib/en57/benchmark.rb @@ -181,57 +181,63 @@ module En57 require_relative "../benchmark/concurrent_append_conflicting_tags" class Runner - def self.classic(runs: 50) - new( - formatter: Table.new, - scenarios: { - "concurrent-append-no-fail-if" => ->( - database_url, - warmup_runs, - measure - ) do - ConcurrentAppendNoFailIf.new( - name: "Concurrent append, no fail_if", - database_url:, - measure:, - warmup_runs:, - runs:, - concurrency: 10, - batch_size: 100, - ) - end, - "concurrent-append-non-conflicting-tags" => ->( - database_url, - warmup_runs, - measure - ) do - ConcurrentAppendNonConflictingTags.new( - name: "Concurrent append, non-conflicting tags", - database_url:, - measure:, - warmup_runs:, - runs:, - concurrency: 10, - batch_size: 100, - ) - end, - "concurrent-append-conflicting-tags" => ->( - database_url, - warmup_runs, - measure - ) do - ConcurrentAppendConflictingTags.new( - name: "Concurrent append, conflicting tags", - database_url:, - measure:, - warmup_runs:, - runs:, - concurrency: 10, - batch_size: 100, - ) - end, - }, - ) + def self.classic(runs: 50, names: nil) + selected_scenarios = scenarios(runs:) + selected_scenarios = selected_scenarios.slice(*names) if names + + new(formatter: Table.new, scenarios: selected_scenarios) + end + + def self.names = scenarios(runs: 1).keys + + def self.scenarios(runs:) + { + "concurrent-append-no-fail-if" => ->( + database_url, + warmup_runs, + measure + ) do + ConcurrentAppendNoFailIf.new( + name: "Concurrent append, no fail_if", + database_url:, + measure:, + warmup_runs:, + runs:, + concurrency: 10, + batch_size: 100, + ) + end, + "concurrent-append-non-conflicting-tags" => ->( + database_url, + warmup_runs, + measure + ) do + ConcurrentAppendNonConflictingTags.new( + name: "Concurrent append, non-conflicting tags", + database_url:, + measure:, + warmup_runs:, + runs:, + concurrency: 10, + batch_size: 100, + ) + end, + "concurrent-append-conflicting-tags" => ->( + database_url, + warmup_runs, + measure + ) do + ConcurrentAppendConflictingTags.new( + name: "Concurrent append, conflicting tags", + database_url:, + measure:, + warmup_runs:, + runs:, + concurrency: 10, + batch_size: 100, + ) + end, + } end def initialize(scenarios:, formatter:) @@ -271,5 +277,46 @@ module En57 end end + class CLI + def initialize( + argv, + out: $stdout, + err: $stderr, + runner: Runner, + runs: Integer(ENV.fetch("BENCHMARK_RUNS", 10)) + ) + @argv = argv + @err = err + @out = out + @runner = runner + @runs = runs + end + + def run + case @argv + in ["list"] + @out.puts(@runner.names) + 0 + in ["run", "all"] + @out.puts(@runner.classic(runs: @runs).run) + 0 + in ["run", name] + return unknown(name) unless @runner.names.include?(name) + + @out.puts(@runner.classic(runs: @runs, names: [name]).run) + 0 + else + @err.puts("Usage: benchmark list | benchmark run NAME | benchmark run all") + 1 + end + end + + private + + def unknown(name) + @err.puts("Unknown benchmark: #{name}") + 1 + end + end end end diff --git a/test/test_benchmark.rb b/test/test_benchmark.rb index b28998e..4a2615a 100644 --- a/test/test_benchmark.rb +++ b/test/test_benchmark.rb @@ -172,6 +172,65 @@ module En57 assert_equal(2, scenario.retry_count) end + def test_cli_lists_available_benchmark_names + runner = Class.new { def self.names = %w[first second] } + output = StringIO.new + + assert_equal(0, CLI.new(["list"], out: output, runner:).run) + assert_equal("first\nsecond\n", output.string) + end + + def test_cli_runs_all_benchmarks + runner = + Class.new do + class << self + attr_reader :classic_args + end + + def self.classic(**kwargs) + @classic_args = kwargs + Data.define(:run).new("results") + end + end + output = StringIO.new + + assert_equal(0, CLI.new(%w[run all], out: output, runner:, runs: 3).run) + assert_equal({ runs: 3 }, runner.classic_args) + assert_equal("results\n", output.string) + end + + def test_cli_runs_named_benchmark + runner = + Class.new do + class << self + attr_reader :classic_args + end + + def self.names = %w[one two] + + def self.classic(**kwargs) + @classic_args = kwargs + Data.define(:run).new("result") + end + end + output = StringIO.new + + assert_equal(0, CLI.new(%w[run one], out: output, runner:, runs: 3).run) + assert_equal({ runs: 3, names: ["one"] }, runner.classic_args) + assert_equal("result\n", output.string) + end + + def test_cli_rejects_unknown_benchmark_name + runner = Class.new { def self.names = %w[one two] } + error = StringIO.new + + assert_equal( + 1, + CLI.new(%w[run unknown], err: error, runner:, runs: 3).run, + ) + assert_equal("Unknown benchmark: unknown\n", error.string) + end + def test_measurement_calculates_summary_statistics measurement = Measurement.from([0.3, 0.1, 0.2])