diff --git a/lib/en57/benchmark.rb b/lib/en57/benchmark.rb
index 9f4636f..bb963f2 100644
--- a/lib/en57/benchmark.rb
+++ b/lib/en57/benchmark.rb
@@ -114,6 +114,7 @@ module En57
         database_url:,
         measure:,
         runs:,
+        warmup_runs:,
         concurrency:,
         batch_size:
       )
@@ -123,19 +124,24 @@ module En57
         @database_url = database_url
         @measure = measure
         @runs = runs
+        @warmup_runs = warmup_runs
       end
 
       attr_reader :name, :runs
 
+
       def run
+        warmup
         @runs.times { call }
         verify
       end
 
       private
 
+      def total_runs = @runs + @warmup_runs
       def call = nil
       def verify = true
+      def warmup = @warmup_runs.times { call }
 
       def concurrently(concurrency)
         Array
@@ -156,32 +162,36 @@ module En57
           scenarios: {
             "concurrent-append-non-conflicting-tags" => ->(
               database_url,
+              warmup_runs,
               measure
             ) do
               ConcurrentAppendNonConflictingTags.new(
                 name: "Concurrent append, non-conflicting tags",
                 database_url:,
                 measure:,
+                warmup_runs:,
                 runs:,
                 concurrency: 10,
                 batch_size: 100,
               )
             end,
-            "concurrent-append-no-fail-if" => ->(database_url, measure) do
+            "concurrent-append-no-fail-if" => ->(database_url, warmup_runs, measure) do
               ConcurrentAppendNoFailIf.new(
                 name: "Concurrent append, no fail_if",
                 database_url:,
                 measure:,
+                warmup_runs:,
                 runs:,
                 concurrency: 10,
                 batch_size: 100,
               )
             end,
-            "concurrent-append-conflicting-tags" => ->(database_url, measure) do
+            "concurrent-append-conflicting-tags" => ->(database_url, warmup_runs, measure) do
               ConcurrentAppendConflictingTags.new(
                 name: "Concurrent append, conflicting tags",
                 database_url:,
                 measure:,
+                warmup_runs:,
                 runs:,
                 concurrency: 10,
                 batch_size: 100,
@@ -204,10 +214,11 @@ module En57
               scenario =
                 mk_scenario.call(
                   server.url,
+                  warmup_runs = 2,
                   ->(&block) { samples << ::Benchmark.realtime { block.call } },
                 )
               verified = scenario.run
-              measurement = Measurement.from(samples)
+              measurement = Measurement.from(samples.drop(warmup_runs))
 
               Result.new(
                 name: scenario.name,
@@ -257,7 +268,8 @@ module En57
       end
 
       def verify =
-        @event_store.read.each.to_a.size == @runs * @concurrency * @batch_size
+        @event_store.read.each.to_a.size ==
+          total_runs * @concurrency * @batch_size
     end
 
     class ConcurrentAppendNonConflictingTags < Scenario
@@ -292,7 +304,8 @@ module En57
       end
 
       def verify =
-        @event_store.read.each.to_a.size == @runs * @concurrency * @batch_size
+        @event_store.read.each.to_a.size ==
+          total_runs * @concurrency * @batch_size
     end
 
     class ConcurrentAppendConflictingTags < Scenario
@@ -331,7 +344,8 @@ module En57
       end
 
       def verify =
-        @event_store.read.each.to_a.size == @runs * @concurrency * @batch_size
+        @event_store.read.each.to_a.size ==
+          total_runs * @concurrency * @batch_size
     end
   end
 end
diff --git a/test/test_benchmark.rb b/test/test_benchmark.rb
index f326682..4dcd5ed 100644
--- a/test/test_benchmark.rb
+++ b/test/test_benchmark.rb
@@ -63,15 +63,15 @@ module En57
 
         server = Data.define(:url).new("postgres://example")
         mk_scenario = ->(name, verified) do
-          ->(_database_url, measure) do
+          ->(_database_url, _warmup_runs, measure) do
             Data
-              .define(:name, :runs, :measure, :verified) do
+              .define(:name, :runs, :measure, :verified, :retry_count) do
                 def run
-                  measure.call { nil }
+                  3.times { measure.call { nil } }
                   verified
                 end
               end
-              .new(name, 1, measure, verified)
+              .new(name, 1, measure, verified, 3)
           end
         end
 
@@ -93,6 +93,59 @@ module En57
         assert_equal(["verified"], formatted_results.map(&:name))
       end
 
+      def test_runner_discards_two_warmup_measurements
+        formatter = Object.new
+        formatted_results = nil
+
+        formatter.define_singleton_method(:format) do |results|
+          formatted_results = results
+          "formatted"
+        end
+
+        scenario_class =
+          Class.new(Scenario) do
+            def initialize(measure:, warmup_runs:)
+              super(
+                name: "warmup",
+                database_url: "postgres://example",
+                measure:,
+                runs: 1,
+                warmup_runs:,
+                concurrency: 1,
+                batch_size: 1,
+              )
+            end
+
+            def call = @measure.call { nil }
+          end
+        server = Data.define(:url).new("postgres://example")
+        durations = [0.1, 0.2, 0.3]
+
+        PgEphemeral.stub(
+          :with_server,
+          ->(instance_name:, &block) { block.call(server) },
+        ) do
+          ::Benchmark.stub(
+            :realtime,
+            ->(&block) do
+              block.call
+              durations.shift
+            end,
+          ) do
+            Runner.new(
+              formatter:,
+              scenarios: {
+                "warmup" => ->(_database_url, warmup_runs, measure) do
+                  scenario_class.new(measure:, warmup_runs:)
+                end,
+              },
+            ).run
+          end
+        end
+
+        assert_equal(0.3, formatted_results.fetch(0).mean)
+      end
+
       def test_measurement_calculates_summary_statistics
         measurement = Measurement.from([0.3, 0.1, 0.2])