From 966ef0e3430ce5064d33e5cc95623b9503ea76c6 Mon Sep 17 00:00:00 2001 From: vshakitskiy Date: Mon, 10 Aug 2026 14:24:32 +0300 Subject: [PATCH] upload the benchmark scripts --- .gitignore | 3 +- benchmark/README.md | 134 +++++++ benchmark/bandit/lib/bandit_bench/router.ex | 52 ++- benchmark/bench.sh | 358 ------------------ benchmark/chatterbox/.gitignore | 2 + benchmark/chatterbox/rebar.config | 7 + benchmark/chatterbox/rebar.lock | 11 + benchmark/chatterbox/run.sh | 6 + .../chatterbox/src/chatterbox_bench.app.src | 11 + .../chatterbox/src/chatterbox_bench_app.erl | 26 ++ .../src/chatterbox_bench_handler.erl | 100 +++++ benchmark/config.sh | 92 +++++ benchmark/elli/src/elli_bench_app.erl | 14 +- benchmark/elli/src/elli_bench_callback.erl | 60 ++- benchmark/ewe@4/src/app.gleam | 132 ++++--- benchmark/ewe@5/manifest.toml | 11 +- benchmark/ewe@5/src/app.gleam | 123 +++--- benchmark/httpd/src/httpd_bench_app.erl | 14 +- benchmark/httpd/src/httpd_bench_callback.erl | 78 ++-- benchmark/latency.sh | 111 ++++++ benchmark/lib.sh | 139 +++++++ benchmark/mist/src/app.gleam | 119 ++++-- benchmark/report.js | 213 +++++++++++ benchmark/results.txt | 224 +++++++++++ .../roadrunner/src/roadrunner_bench_app.erl | 16 +- .../src/roadrunner_bench_callback.erl | 60 +-- .../roadrunner/src/roadrunner_bench_sup.erl | 26 +- benchmark/throughput.sh | 135 +++++++ benchmark/wrk/get_headers.lua | 5 + benchmark/wrk/post_echo.lua | 6 - benchmark/wrk/post_echo_10kb.lua | 3 + benchmark/wrk/post_echo_1kb.lua | 3 + benchmark/wrk/post_echo_1kb_headers.lua | 8 + ...chunked.lua => post_echo_chunked_10kb.lua} | 14 +- benchmark/wrk2-setup.sh | 8 - src/ewe/internal/sse_ffi.erl | 17 +- 36 files changed, 1662 insertions(+), 679 deletions(-) create mode 100644 benchmark/README.md delete mode 100755 benchmark/bench.sh create mode 100644 benchmark/chatterbox/.gitignore create mode 100644 benchmark/chatterbox/rebar.config create mode 100644 benchmark/chatterbox/rebar.lock create mode 100755 benchmark/chatterbox/run.sh create mode 100644 benchmark/chatterbox/src/chatterbox_bench.app.src create mode 100644 benchmark/chatterbox/src/chatterbox_bench_app.erl create mode 100644 benchmark/chatterbox/src/chatterbox_bench_handler.erl create mode 100644 benchmark/config.sh create mode 100755 benchmark/latency.sh create mode 100644 benchmark/lib.sh create mode 100755 benchmark/report.js create mode 100644 benchmark/results.txt create mode 100755 benchmark/throughput.sh create mode 100644 benchmark/wrk/get_headers.lua delete mode 100644 benchmark/wrk/post_echo.lua create mode 100644 benchmark/wrk/post_echo_10kb.lua create mode 100644 benchmark/wrk/post_echo_1kb.lua create mode 100644 benchmark/wrk/post_echo_1kb_headers.lua rename benchmark/wrk/{post_echo_chunked.lua => post_echo_chunked_10kb.lua} (53%) diff --git a/.gitignore b/.gitignore index ae0b8c8..f322a40 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,7 @@ autobahn/server /bench /dev/priv/file_1gb.bin /dev/priv/file_100kb.bin -/benchmark/priv/file_1gb.bin -/benchmark/priv/file_100kb.bin +/benchmark/priv/*.bin /benchmark/results /benchmark/.wrk2 NOTE.md \ No newline at end of file diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..a42916d --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,134 @@ +Compares ewe against other BEAM web servers on the same set of endpoints over +both HTTP/1.1 and h2c. + +Every server implements the same routes so a row of the report is the same work +done by different web servers. Where a server has no native API for a case its +route is absent and the case is skipped rather than hand rolled. + +## Requirements + +To run everything you should have Gleam, Elixir and Erlang installed. For +benchmarks the tools are `wrk2` (can be built by `./wrk2-setup.sh`) and +`h2load` from `nghttp2`. For printing the reports I have a `report.js` that I run +with Deno. + +## Quick start + +```sh +./throughput.sh # runs full matrix and its like ~80 min +./latency.sh # runs full ladder for ~50 min + +./throughput.sh --servers ewe@5,roadrunner --cases hello,sse_big +./latency.sh --cases hello +``` + +Both write a timestamped directory under `results/` and print a report at the +end. To rerender one later: + +```sh +./report.js results/-throughput/throughput.csv +./report.js results/-latency/latency.csv # you can also run both at once +``` + +## Servers + +| name | dir | http/1 | h2c | +| --- | --- | --- | --- | --- | +| `ewe@5` | `ewe@5` | 3006 | 3006 | +| `ewe@4` | `ewe@4` | 3001 | — | +| `mist` | `mist` | 3002 | — | +| `elli` | `elli` | 3003 | — | +| `bandit` | `bandit` | 3004 | 3004 | +| `httpd` | `httpd` | 3005 | — | +| `roadrunner` | `roadrunner` | 3007 | 3008 | +| `chatterbox` | `chatterbox` | — | 8082 | + +Each is started fresh for its measurements and stopped afterwards. `run.sh` in +each directory is the start command, the gleam ones use `gleam run`. + +There are fifteen cases. +| case | request | response | +| --- | --- | --- | --- | +| `hello` | `GET /hello` | `Hello, Joe!`, 11 B | +| `hello_headers` | `GET /hello` + 7 headers | 11 B | +| `echo_1kb` | `POST /echo`, 1 KiB body | echoes body | +| `echo_1kb_headers` | `POST /echo`, 1 KiB + 7 headers | echoes body | +| `echo_10kb` | `POST /echo`, 10 KiB body | echoes body | +| `echo_chunked_10kb` | `POST /echo/chunked`, 10 KiB chunked | echoes body | +| `file_tiny` | `GET /file/tiny` | `priv/file_1kb.bin`, 1 KiB | +| `file_small` | `GET /file/small` | `priv/file_100kb.bin`, 100 KiB | +| `file_big` | `GET /file/big` | `priv/file_5mb.bin`, 5 MiB | +| `stream` | `GET /stream` | `hello, ` then `Joe!`, 11 B | +| `stream_small` | `GET /stream/small` | 100 × 64 B chunks, 6,400 B | +| `stream_big` | `GET /stream/big` | 64 × 16 KiB chunks, 1 MiB | +| `sse` | `GET /sse` | 32 events, 2,688 B | +| `sse_small` | `GET /sse/small` | 100 events, 8,400 B | +| `sse_big` | `GET /sse/big` | 64 events of 16 KiB, 1,049,856 B | + +SSE events are `event: tick` with a 64-byte data line (and 16 KiB for `sse_big`), +framed by each server's own SSE encoder on every request. The header set is +seven fields including three cookies. + +There is not native SSE API in `bandit`, `chatterbox`, `elli`, `httpd`, so +comparing with streaming endpoints is enough. And `elli`, `httpd` do not have +incremental request body read, so we can't measure chunked reading. + +## Scripts + +**`throughput.sh`**: requests a second, per server, per case and per protocol. +Drives both protocols with `h2load` so the h1 and h2 rows are the same tool +measuring the same work. Reports no coordinated omission correction which is +why there is separate latency script. Accepts `--servers`, `--cases` and `--profiles`. + +**`latency.sh`**: response latency under fixed offered HTTP/1 only load. `wrk2` +holds a target rate rather than a connection count so its percentiles include +the time a request waited to be sent. HTTP/1 only because wrk2 has no HTTP/2 +support and an h2 column from a different tool would not be comparable. Accepts `--servers` and `--cases`. + +**`report.js`**: small script that renders either CSV as tables. Detects the +format from the header so it takes any mix of files. Throughput mode prints req/s and +messages/s then warnings for servers answering incorrectly, servers collapsed +against their peers and cells whose repeats disagreed by over 5%. + +**`config.sh`**: settings on what to measure: servers, cases, profiles, latency rates, +unsupported combinations. + +**`lib.sh`** — settings on how to measure things: fixture generation, port handling, server +start/stop, core pinning. + +**`wrk2-setup.sh`**: clones and builds wrk2 into `.wrk2/`. + +The environment variables are: + +``` +CONNECTIONS=50 THREADS=4 DURATION=10 WARMUP=3 WARMUP_RATE=20000 +REPEATS=3 H2_STREAMS=10 WRK2= +``` + +Latency uses fixed rate ladders per case (`latency_rates()` in `config.sh`) so +every server is driven at the same absolute load to make servers really comparable. + +If the machine has four or more cores the server gets the first half and the +load generator the second since the generator is heavy enough to fight the +server for CPU otherwise. The chosen ranges are printed at the top of each run. + +The output of the scripts is something like this: + +``` +results/-throughput/ + throughput.csv # server,profile,protocol,connections,streams, + # case,repeat,requests_per_sec,messages, + # messages_per_sec,mb_per_sec,succeeded,failed,non_2xx + run.txt # date, load settings, cpu pinning + report.txt # rendered tables + .log # server stdout and stderr + ______.txt # raw h2load output + +results/-latency/ + latency.csv # server,case,target_rate,achieved_rate, + # p50,p90,p99,p999,p50_raw,p99_raw,errors + run.txt + report.txt + .log + ____.txt # raw wrk2 output +``` \ No newline at end of file diff --git a/benchmark/bandit/lib/bandit_bench/router.ex b/benchmark/bandit/lib/bandit_bench/router.ex index dbf8f3a..41f5a32 100644 --- a/benchmark/bandit/lib/bandit_bench/router.ex +++ b/benchmark/bandit/lib/bandit_bench/router.ex @@ -1,7 +1,12 @@ defmodule BanditBench.Router do use Plug.Router - @sse_events 32 + @small_count 100 + @big_count 64 + @big_repeats 256 + + @small_line "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + @big_line String.duplicate(@small_line, @big_repeats) plug(:match) plug(:dispatch) @@ -27,12 +32,18 @@ defmodule BanditBench.Router do conn end - get "/sse" do + get "/stream/small" do + conn |> send_chunked(200) |> stream_burst(@small_line, @small_count) + end + + get "/stream/big" do + conn |> send_chunked(200) |> stream_burst(@big_line, @big_count) + end + + get "/file/tiny" do conn - |> put_resp_header("content-type", "text/event-stream") - |> put_resp_header("cache-control", "no-cache") - |> send_chunked(200) - |> sse_burst() + |> put_resp_header("content-type", "application/octet-stream") + |> send_file(200, "../priv/file_1kb.bin") end get "/file/small" do @@ -44,39 +55,22 @@ defmodule BanditBench.Router do get "/file/big" do conn |> put_resp_header("content-type", "application/octet-stream") - |> send_file(200, "../priv/file_1gb.bin") + |> send_file(200, "../priv/file_5mb.bin") end match _ do send_resp(conn, 404, "") end - defp sse_burst(conn) do - send(self(), {:sse_tick, 1}) - sse_loop(conn) - end - - defp sse_loop(conn) do - receive do - {:sse_tick, n} when n > @sse_events -> - conn - - {:sse_tick, n} -> - case chunk(conn, sse_event(n)) do - {:ok, conn} -> - send(self(), {:sse_tick, n + 1}) - sse_loop(conn) + defp stream_burst(conn, _data, 0), do: conn - {:error, _reason} -> - conn - end + defp stream_burst(conn, data, remaining) do + case chunk(conn, data) do + {:ok, conn} -> stream_burst(conn, data, remaining - 1) + {:error, _reason} -> conn end end - defp sse_event(n) do - "event: tick\nid: #{n}\ndata: {\"n\":#{n},\"at\":\"benchmark\"}\n\n" - end - defp echo_chunked({:more, partial, conn}, acc) do Plug.Conn.read_body(conn, length: 4096) |> echo_chunked(acc <> partial) end diff --git a/benchmark/bench.sh b/benchmark/bench.sh deleted file mode 100755 index efc7593..0000000 --- a/benchmark/bench.sh +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env bash -# -# Drives wrk2 against every server in benchmark/. For each endpoint, it first -# probes for the server's saturation throughput, then re-runs at fixed target -# rates that are 50/75/90/95% of that saturation point, recording both corrected -# and uncorrected latency percentiles at each rate. -# -# Requires benchmark/.wrk2/wrk2, so run ./wrk2-setup.sh once first. -# -# Usage: -# ./bench.sh [--servers "ewe@5,mist"] [--endpoints "sse"] [--pcts "50 75 90 95"] -# [--conns 50] [--duration 20s] [--warmup 5s] [--threads 4] -# [--probe-rate 500000] [--probe-duration 5s] -# -# --servers comma separated subset of server names (default: all) -# --endpoints comma separated subset of endpoint names (default: all) -# --pcts space separated percentages of saturation to test (default: "50 75 90 95") -# --conns connections held open per run (default: 50) -# --duration wrk2 measured run duration (default: 20s) -# --warmup untimed warmup before measuring each server (default: 5s) -# --threads wrk2 threads (default: 4) -# --probe-rate -R used for the saturation probe (default: 500000) -# --probe-duration duration of the saturation probe (default: 5s) - -set -u - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$ROOT" - -# name|port|dir|start command -SERVERS=( - "ewe@5|3006|ewe@5|gleam run" - "ewe@4|3001|ewe@4|gleam run" - "mist|3002|mist|gleam run" - "elli|3003|elli|./run.sh" - "bandit|3004|bandit|./run.sh" - "httpd|3005|httpd|./run.sh" - "roadrunner|3007|roadrunner|./run.sh" -) - -# name|path|wrk script relative to benchmark/wrk -ENDPOINTS=( - "hello|/hello|" - "echo|/echo|post_echo.lua" - "echo_chunked|/echo/chunked|post_echo_chunked.lua" - "stream|/stream|" - "sse|/sse|" - "file_small|/file/small|" -) - -SSE_EVENTS=32 - -# server|endpoint|reason combos to skip before probing. Two distinct reasons: -# - not_implemented: the server doesn't implement the feature the endpoint is -# meant to exercise, so measuring it tells us nothing. -# -# - unstable: the endpoint collapses under wrk2's rate-limited load for a -# confirmed reproducible reason that isn't a benchmark setup mistake. -SKIP=( - "elli|echo_chunked|not_implemented" - "httpd|echo_chunked|not_implemented" - "elli|stream|unstable" - "elli|sse|unstable" - "mist|stream|unstable" - "ewe@4|stream|unstable" - "ewe@4|file_small|unstable" -) - -skip_endpoint() { - local name="$1" ename="$2" entry s_name s_ename - for entry in "${SKIP[@]}"; do - IFS='|' read -r s_name s_ename SKIP_REASON <<< "$entry" - if [ "$s_name" = "$name" ] && [ "$s_ename" = "$ename" ]; then - return 0 - fi - done - return 1 -} - -server_selected() { - local name="$1" - [ -z "${ONLY_SERVERS:-}" ] && return 0 - case ",$ONLY_SERVERS," in - *",$name,"*) return 0 ;; - *) return 1 ;; - esac -} - -endpoint_selected() { - local name="$1" - [ -z "${ONLY_ENDPOINTS:-}" ] && return 0 - case ",$ONLY_ENDPOINTS," in - *",$name,"*) return 0 ;; - *) return 1 ;; - esac -} - -# Kills whatever is already listening on $1, e.g. a server orphaned by a -# previous run that got interrupted before its own cleanup ran. -free_port() { - local port="$1" pid - pid=$(ss -tlnp 2>/dev/null | awk -v p=":$port\$" '$4 ~ p' | grep -oP 'pid=\K[0-9]+' | head -1) - if [ -n "$pid" ]; then - echo " port $port already in use by pid $pid, killing" >&2 - kill -9 "$pid" 2>/dev/null - sleep 1 - fi -} - -# Cleans up whichever server is currently running if the script exits early. -CURRENT_PGID="" -cleanup() { - if [ -n "$CURRENT_PGID" ]; then - kill -TERM -- -"$CURRENT_PGID" 2>/dev/null - sleep 1 - kill -KILL -- -"$CURRENT_PGID" 2>/dev/null - fi -} -trap cleanup EXIT INT TERM - -wait_ready() { - local port="$1" tries=60 - while [ "$tries" -gt 0 ]; do - if curl -s -o /dev/null --max-time 1 "http://127.0.0.1:$port/hello"; then - return 0 - fi - sleep 0.5 - tries=$((tries - 1)) - done - return 1 -} - -# Starts $dir/$cmd in its own process group, logs to $log, and waits for $port -# to answer. Must NOT be called via $(...). Sets CURRENT_PGID on success clears -# it on failure. Returns 0/1. -start_server() { - local dir="$1" cmd="$2" port="$3" log="$4" - - free_port "$port" - - ( cd "$ROOT/$dir" && exec setsid $cmd > "$log" 2>&1 < /dev/null ) & - CURRENT_PGID=$! - - if ! wait_ready "$port"; then - echo " failed to start, see $log" >&2 - kill -KILL -- -"$CURRENT_PGID" 2>/dev/null - wait "$CURRENT_PGID" 2>/dev/null - CURRENT_PGID="" - return 1 - fi - - return 0 -} - -stop_server() { - local pgid="$1" - kill -TERM -- -"$pgid" 2>/dev/null - sleep 1 - kill -KILL -- -"$pgid" 2>/dev/null - wait "$pgid" 2>/dev/null - CURRENT_PGID="" - sleep 1 -} - -WRK2_BIN="${WRK2_BIN:-$ROOT/.wrk2/wrk2}" -if ! [ -x "$WRK2_BIN" ]; then - WRK2_BIN="$(command -v wrk2 || true)" -fi -if [ -z "$WRK2_BIN" ] || ! [ -x "$WRK2_BIN" ]; then - echo "wrk2 not found. Run ./wrk2-setup.sh first (or set WRK2_BIN)." >&2 - exit 1 -fi - -PCTS="50 75 90 95" -CONNS=50 -DURATION="20s" -WARMUP_DURATION="5s" -THREADS=4 -PROBE_RATE=500000 -PROBE_DURATION="5s" -ONLY_SERVERS="" -ONLY_ENDPOINTS="" - -while [ $# -gt 0 ]; do - case "$1" in - --servers) ONLY_SERVERS="$2"; shift 2 ;; - --endpoints) ONLY_ENDPOINTS="$2"; shift 2 ;; - --pcts) PCTS="$2"; shift 2 ;; - --conns) CONNS="$2"; shift 2 ;; - --duration) DURATION="$2"; shift 2 ;; - --warmup) WARMUP_DURATION="$2"; shift 2 ;; - --threads) THREADS="$2"; shift 2 ;; - --probe-rate) PROBE_RATE="$2"; shift 2 ;; - --probe-duration) PROBE_DURATION="$2"; shift 2 ;; - *) echo "Unknown argument: $1" >&2; exit 1 ;; - esac -done - -TIMESTAMP="$(date +%Y%m%d-%H%M%S)" -RESULTS_DIR="$ROOT/results/$TIMESTAMP-wrk2" -mkdir -p "$RESULTS_DIR" -CSV="$RESULTS_DIR/results.csv" -echo "server,endpoint,pct_of_saturation,target_rate,saturation_rate,connections,threads,duration,requests_per_sec,corrected_p50,corrected_p90,corrected_p99,corrected_p999,corrected_p9999,uncorrected_p50,uncorrected_p90,uncorrected_p99,uncorrected_p999,uncorrected_p9999,errors" > "$CSV" - -mkdir -p "$ROOT/priv" -[ -f "$ROOT/priv/file_100kb.bin" ] || head -c 100K /dev/urandom > "$ROOT/priv/file_100kb.bin" - -wrk2_url() { - local port="$1" path="$2" - echo "http://127.0.0.1:$port$path" -} - -wrk2_run() { - # Runs wrk2 and prints raw output; $1=port $2=path $3=script $4=rate $5=conn - # $6=duration - local port="$1" path="$2" script="$3" rate="$4" conn="$5" duration="$6" - if [ -n "$script" ]; then - "$WRK2_BIN" -t"$THREADS" -c"$conn" -d"$duration" -R"$rate" -U \ - -s "$ROOT/wrk/$script" "$(wrk2_url "$port" "$path")" - else - "$WRK2_BIN" -t"$THREADS" -c"$conn" -d"$duration" -R"$rate" -U \ - "$(wrk2_url "$port" "$path")" - fi -} - -# Probes achievable throughput by requesting a rate far above what the server -# can sustain and reading back what it actually achieved. -probe_saturation() { - local port="$1" path="$2" script="$3" - local out rps - out=$(wrk2_run "$port" "$path" "$script" "$PROBE_RATE" "$CONNS" "$PROBE_DURATION") - rps=$(printf '%s\n' "$out" | awk '/Requests\/sec:/ { print $2 }') - # Integer floor; falls back to 1 if parsing failed so a /0 can't happen. - printf '%.0f\n' "${rps:-1}" -} - -# Parses wrk2 -U output into pipe-separated fields. -parse_wrk2() { - awk ' - /Latency Distribution \(HdrHistogram - Recorded Latency\)/ { section="c"; next } - /Latency Distribution \(HdrHistogram - Uncorrected Latency/ { section="u"; next } - section=="c" && / 50\.000%/ { c50=$2 } - section=="c" && / 90\.000%/ { c90=$2 } - section=="c" && / 99\.000%/ { c99=$2 } - section=="c" && / 99\.900%/ { c999=$2 } - section=="c" && / 99\.990%/ { c9999=$2 } - section=="u" && / 50\.000%/ { u50=$2 } - section=="u" && / 90\.000%/ { u90=$2 } - section=="u" && / 99\.000%/ { u99=$2 } - section=="u" && / 99\.900%/ { u999=$2 } - section=="u" && / 99\.990%/ { u9999=$2 } - /Requests\/sec:/ { rps=$2 } - /Socket errors:/ { gsub(",", "", $0); err=$0 } - END { - printf "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n", \ - rps, c50, c90, c99, c999, c9999, u50, u90, u99, u999, u9999, err - } - ' -} - -# Converts a wrk-style duration string to microseconds, for computing the -# corrected/uncorrected ratio. -to_us() { - awk -v v="$1" ' - BEGIN { - if (v ~ /us$/) { sub(/us$/, "", v); printf "%.4f", v } - else if (v ~ /ms$/) { sub(/ms$/, "", v); printf "%.4f", v * 1000 } - else if (v ~ /s$/) { sub(/s$/, "", v); printf "%.4f", v * 1000000 } - else { printf "%.4f", v } - } - ' -} - -# corrected/uncorrected, as like "12.3x". -ratio_of() { - local c u - c=$(to_us "$1") - u=$(to_us "$2") - awk -v c="$c" -v u="$u" 'BEGIN { if (u + 0 == 0) print "n/a"; else printf "%.1fx", c / u }' -} - -table_header() { - printf " %-5s %11s %11s %-24s %-24s %s\n" \ - "load" "target/s" "actual/s" "corrected p50/p99/p99.9" "uncorrected p50/p99/p99.9" "CO ratio(p99)" -} - -run_rate() { - local server="$1" port="$2" endpoint="$3" path="$4" script="$5" pct="$6" sat="$7" - local rate=$(( sat * pct / 100 )) - [ "$rate" -lt 1 ] && rate=1 - - local raw="$RESULTS_DIR/${server}__${endpoint}__p${pct}.txt" - local out - out=$(wrk2_run "$port" "$path" "$script" "$rate" "$CONNS" "$DURATION" | tee "$raw") - - local parsed rps c50 c90 c99 c999 c9999 u50 u90 u99 u999 u9999 err - parsed=$(printf '%s\n' "$out" | parse_wrk2) - IFS='|' read -r rps c50 c90 c99 c999 c9999 u50 u90 u99 u999 u9999 err <<< "$parsed" - - local err_csv="" - [ -n "$err" ] && err_csv="\"$err\"" - echo "$server,$endpoint,$pct,$rate,$sat,$CONNS,$THREADS,$DURATION,$rps,$c50,$c90,$c99,$c999,$c9999,$u50,$u90,$u99,$u999,$u9999,$err_csv" >> "$CSV" - - local ratio corrected uncorrected - ratio=$(ratio_of "$c99" "$u99") - corrected="$c50/$c99/$c999" - uncorrected="$u50/$u99/$u999" - printf " %3d%% %11s %11s %-24s %-24s %s\n" \ - "$pct" "$rate" "$rps" "$corrected" "$uncorrected" "$ratio" - [ -n "$err" ] && printf " %s\n" "$err" -} - -for entry in "${SERVERS[@]}"; do - IFS='|' read -r name port dir cmd <<< "$entry" - server_selected "$name" || continue - - echo "=== $name (port $port) ===" - server_log="$RESULTS_DIR/${name}.server.log" - - start_server "$dir" "$cmd" "$port" "$server_log" || continue - pgid="$CURRENT_PGID" - - wrk -t2 -c20 -d"$WARMUP_DURATION" "http://127.0.0.1:$port/hello" > /dev/null 2>&1 - - for endpoint_entry in "${ENDPOINTS[@]}"; do - IFS='|' read -r ename epath escript <<< "$endpoint_entry" - endpoint_selected "$ename" || continue - - if skip_endpoint "$name" "$ename"; then - echo - case "$SKIP_REASON" in - not_implemented) - echo " $ename (skipped: $name doesn't implement this feature)" ;; - unstable) - echo " $ename (skipped: cannot measure properly)" ;; - esac - continue - fi - - saturation=$(probe_saturation "$port" "$epath" "$escript") - echo - if [ "$ename" = "sse" ]; then - echo " $ename (saturation ~$saturation streams/s = ~$(( saturation * SSE_EVENTS )) events/s)" - else - echo " $ename (saturation ~$saturation req/s)" - fi - table_header - - for pct in $PCTS; do - run_rate "$name" "$port" "$ename" "$epath" "$escript" "$pct" "$saturation" - done - done - - stop_server "$pgid" -done - -echo -echo "Results: $CSV" diff --git a/benchmark/chatterbox/.gitignore b/benchmark/chatterbox/.gitignore new file mode 100644 index 0000000..257d677 --- /dev/null +++ b/benchmark/chatterbox/.gitignore @@ -0,0 +1,2 @@ +_build/ +erl_crash.dump diff --git a/benchmark/chatterbox/rebar.config b/benchmark/chatterbox/rebar.config new file mode 100644 index 0000000..4b57e5b --- /dev/null +++ b/benchmark/chatterbox/rebar.config @@ -0,0 +1,7 @@ +{deps, [ + {chatterbox, "0.8.0"} +]}. + +{shell, [ + {apps, [chatterbox_bench]} +]}. diff --git a/benchmark/chatterbox/rebar.lock b/benchmark/chatterbox/rebar.lock new file mode 100644 index 0000000..94acb13 --- /dev/null +++ b/benchmark/chatterbox/rebar.lock @@ -0,0 +1,11 @@ +{"1.2.0", +[{<<"chatterbox">>,{pkg,<<"chatterbox">>,<<"0.8.0">>},0}, + {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},1}]}. +[ +{pkg_hash,[ + {<<"chatterbox">>, <<"AF8B95EF45D5B9783B67ECEA1E23A4CCE058FCE5E950B42F84F4DF8BC7620D0B">>}, + {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}]}, +{pkg_hash_ext,[ + {<<"chatterbox">>, <<"70095D965FFE37509C5FDFA83CE28E31879A3E59C8438288989FC92112539423">>}, + {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}]} +]. diff --git a/benchmark/chatterbox/run.sh b/benchmark/chatterbox/run.sh new file mode 100755 index 0000000..3cbb814 --- /dev/null +++ b/benchmark/chatterbox/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +rebar3 compile +erl -pa _build/default/lib/*/ebin \ + -eval "application:ensure_all_started(chatterbox_bench)" \ + -noshell diff --git a/benchmark/chatterbox/src/chatterbox_bench.app.src b/benchmark/chatterbox/src/chatterbox_bench.app.src new file mode 100644 index 0000000..ccfb180 --- /dev/null +++ b/benchmark/chatterbox/src/chatterbox_bench.app.src @@ -0,0 +1,11 @@ +{application, chatterbox_bench, + [{description, "Chatterbox HTTP/2 benchmark"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {chatterbox_bench_app, []}}, + {applications, [kernel, stdlib, chatterbox]}, + {env, []}, + {modules, []}, + {licenses, ["Unlicense"]}, + {links, []} + ]}. diff --git a/benchmark/chatterbox/src/chatterbox_bench_app.erl b/benchmark/chatterbox/src/chatterbox_bench_app.erl new file mode 100644 index 0000000..a200ed6 --- /dev/null +++ b/benchmark/chatterbox/src/chatterbox_bench_app.erl @@ -0,0 +1,26 @@ +-module(chatterbox_bench_app). +-behaviour(application). + +-export([start/2, stop/1, payload/1]). + +-define(SMALL_LINE, <<"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef">>). +-define(BIG_REPEATS, 256). + +start(_StartType, _StartArgs) -> + application:set_env(chatterbox, port, 8082), + application:set_env(chatterbox, ssl, false), + application:set_env(chatterbox, concurrent_acceptors, 100), + application:set_env(chatterbox, stream_callback_mod, chatterbox_bench_handler), + BigLine = binary:copy(?SMALL_LINE, ?BIG_REPEATS), + put_payload(small_chunk, ?SMALL_LINE), + put_payload(big_chunk, BigLine), + chatterbox_sup:start_link(). + +stop(_State) -> + ok. + +put_payload(Key, Value) -> + persistent_term:put({?MODULE, Key}, Value). + +payload(Key) -> + persistent_term:get({?MODULE, Key}). diff --git a/benchmark/chatterbox/src/chatterbox_bench_handler.erl b/benchmark/chatterbox/src/chatterbox_bench_handler.erl new file mode 100644 index 0000000..8a941c3 --- /dev/null +++ b/benchmark/chatterbox/src/chatterbox_bench_handler.erl @@ -0,0 +1,100 @@ +-module(chatterbox_bench_handler). + +-behaviour(h2_stream). + +-export([ + init/3, + on_receive_request_headers/2, + on_send_push_promise/2, + on_receive_request_data/2, + on_request_end_stream/1 +]). + +-define(SMALL_COUNT, 100). +-define(BIG_COUNT, 64). + +-record(state, { + conn_pid :: pid(), + stream_id :: non_neg_integer(), + path = undefined :: binary() | undefined, + buffer = <<>> :: binary() +}). + +init(ConnPid, StreamId, _Opts) -> + {ok, #state{conn_pid = ConnPid, stream_id = StreamId}}. + +on_receive_request_headers(Headers, State) -> + Path = proplists:get_value(<<":path">>, Headers), + {ok, State#state{path = Path}}. + +on_send_push_promise(_Headers, State) -> + {ok, State}. + +on_receive_request_data(Data, State = #state{buffer = Buffer}) -> + {ok, State#state{buffer = <>}}. + +on_request_end_stream( + State = #state{conn_pid = ConnPid, stream_id = StreamId, path = Path, buffer = Buffer} +) -> + respond(Path, Buffer, ConnPid, StreamId, State). + +respond(<<"/hello">>, _Body, ConnPid, StreamId, State) -> + send(ConnPid, StreamId, 200, <<"text/plain">>, <<"Hello, Joe!">>), + {ok, State}; +respond(<<"/echo">>, Body, ConnPid, StreamId, State) -> + send(ConnPid, StreamId, 200, <<"application/octet-stream">>, Body), + {ok, State}; +respond(<<"/echo/chunked">>, Body, ConnPid, StreamId, State) -> + send(ConnPid, StreamId, 200, <<"application/octet-stream">>, Body), + {ok, State}; +respond(<<"/file/tiny">>, _Body, ConnPid, StreamId, State) -> + send_file(ConnPid, StreamId, "../priv/file_1kb.bin"), + {ok, State}; +respond(<<"/file/small">>, _Body, ConnPid, StreamId, State) -> + send_file(ConnPid, StreamId, "../priv/file_100kb.bin"), + {ok, State}; +respond(<<"/file/big">>, _Body, ConnPid, StreamId, State) -> + send_file(ConnPid, StreamId, "../priv/file_5mb.bin"), + {ok, State}; +respond(<<"/stream">>, _Body, ConnPid, StreamId, State) -> + send_headers(ConnPid, StreamId, [{<<"content-type">>, <<"text/plain">>}]), + h2_connection:send_body(ConnPid, StreamId, <<"hello, ">>, [{send_end_stream, false}]), + h2_connection:send_body(ConnPid, StreamId, <<"Joe!">>), + {ok, State}; +respond(<<"/stream/small">>, _Body, ConnPid, StreamId, State) -> + send_stream(ConnPid, StreamId, <<"text/plain">>, small_chunk, ?SMALL_COUNT), + {ok, State}; +respond(<<"/stream/big">>, _Body, ConnPid, StreamId, State) -> + send_stream(ConnPid, StreamId, <<"application/octet-stream">>, big_chunk, ?BIG_COUNT), + {ok, State}; +respond(_Path, _Body, ConnPid, StreamId, State) -> + send(ConnPid, StreamId, 404, <<"text/plain">>, <<"not found">>), + {ok, State}. + +send_file(ConnPid, StreamId, Path) -> + case file:read_file(Path) of + {ok, Data} -> send(ConnPid, StreamId, 200, <<"application/octet-stream">>, Data); + {error, _Reason} -> send(ConnPid, StreamId, 404, <<"text/plain">>, <<"not found">>) + end. + +send_headers(ConnPid, StreamId, Headers) -> + h2_connection:send_headers(ConnPid, StreamId, [{<<":status">>, <<"200">>} | Headers]). + +send(ConnPid, StreamId, Status, ContentType, Body) -> + Headers = [ + {<<":status">>, integer_to_binary(Status)}, + {<<"content-type">>, ContentType}, + {<<"content-length">>, integer_to_binary(iolist_size(Body))} + ], + h2_connection:send_headers(ConnPid, StreamId, Headers), + h2_connection:send_body(ConnPid, StreamId, iolist_to_binary(Body)). + +send_stream(ConnPid, StreamId, ContentType, Key, Count) -> + send_headers(ConnPid, StreamId, [{<<"content-type">>, ContentType}]), + send_chunks(ConnPid, StreamId, chatterbox_bench_app:payload(Key), Count). + +send_chunks(ConnPid, StreamId, Chunk, 1) -> + h2_connection:send_body(ConnPid, StreamId, Chunk); +send_chunks(ConnPid, StreamId, Chunk, N) -> + h2_connection:send_body(ConnPid, StreamId, Chunk, [{send_end_stream, false}]), + send_chunks(ConnPid, StreamId, Chunk, N - 1). diff --git a/benchmark/config.sh b/benchmark/config.sh new file mode 100644 index 0000000..bd2115e --- /dev/null +++ b/benchmark/config.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# name|directory|start command|http/1 port|http/2 port +SERVERS=( + "ewe@5|ewe@5|gleam run|3006|3006" + "ewe@4|ewe@4|gleam run|3001|-" + "mist|mist|gleam run|3002|-" + "elli|elli|./run.sh|3003|-" + "bandit|bandit|./run.sh|3004|3004" + "httpd|httpd|./run.sh|3005|-" + "roadrunner|roadrunner|./run.sh|3007|3008" + "chatterbox|chatterbox|./run.sh|-|8082" +) + +# name|path|body|headers|messages +CASES=( + "hello|/hello|-|no|1" + "hello_headers|/hello|-|yes|1" + "echo_1kb|/echo|1kb|no|1" + "echo_1kb_headers|/echo|1kb|yes|1" + "echo_10kb|/echo|10kb|no|1" + "echo_chunked_10kb|/echo/chunked|10kb|no|1" + "file_tiny|/file/tiny|-|no|1" + "file_small|/file/small|-|no|1" + "file_big|/file/big|-|no|1" + "stream|/stream|-|no|2" + "stream_small|/stream/small|-|no|100" + "stream_big|/stream/big|-|no|64" + "sse|/sse|-|no|32" + "sse_small|/sse/small|-|no|100" + "sse_big|/sse/big|-|no|64" +) + +unsupported() { + case "$1:$2" in + bandit:sse* | chatterbox:sse* | elli:sse* | httpd:sse*) return 0 ;; + elli:echo_chunked_10kb | httpd:echo_chunked_10kb) return 0 ;; + *) return 1 ;; + esac +} + +CONNECTIONS="${CONNECTIONS:-50}" +THREADS="${THREADS:-4}" +DURATION="${DURATION:-10}" +WARMUP="${WARMUP:-3}" +WARMUP_RATE="${WARMUP_RATE:-20000}" +REPEATS="${REPEATS:-3}" + +# name|protocol|connections|streams +PROFILES=( + "h1|h1|$CONNECTIONS|1" + "h2|h2|$CONNECTIONS|${H2_STREAMS:-10}" + "h2-serial|h2|$CONNECTIONS|1" +) + +DEFAULT_PROFILES="h1,h2" + +HEADER_ARGS=( + -H "cookie: session=abc123def456" + -H "cookie: theme=dark" + -H "cookie: locale=en-US" + -H "x-request-id: 9f86d081-b1bb-4c2e-8f21-9c7b1f2a0e3e" + -H "x-client-version: 4.2.1" + -H "accept-language: en-US,en;q=0.9" + -H "x-forwarded-for: 203.0.113.42" +) + +latency_rates() { + case "$1" in + hello | hello_headers | echo_1kb) echo "50000 100000 150000" ;; + echo_1kb_headers) echo "40000 80000 120000" ;; + echo_10kb) echo "20000 60000 100000" ;; + echo_chunked_10kb) echo "40000 70000 100000" ;; + file_tiny | stream) echo "20000 40000 60000" ;; + file_small) echo "15000 30000 45000" ;; + file_big) echo "500 1000 1500" ;; + sse) echo "6000 10000 14000" ;; + stream_small | stream_big | sse_small | sse_big) echo "3000 5000 7000" ;; + *) echo "1000" ;; + esac +} + +wrk_script() { + case "$1" in + hello_headers) echo "get_headers.lua" ;; + echo_1kb) echo "post_echo_1kb.lua" ;; + echo_1kb_headers) echo "post_echo_1kb_headers.lua" ;; + echo_10kb) echo "post_echo_10kb.lua" ;; + echo_chunked_10kb) echo "post_echo_chunked_10kb.lua" ;; + *) echo "" ;; + esac +} diff --git a/benchmark/elli/src/elli_bench_app.erl b/benchmark/elli/src/elli_bench_app.erl index b4099ac..2f49b40 100644 --- a/benchmark/elli/src/elli_bench_app.erl +++ b/benchmark/elli/src/elli_bench_app.erl @@ -2,10 +2,22 @@ -behaviour(application). --export([start/2, stop/1]). +-export([start/2, stop/1, payload/1]). + +-define(SMALL_LINE, <<"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef">>). +-define(BIG_REPEATS, 256). start(_StartType, _StartArgs) -> + BigLine = binary:copy(?SMALL_LINE, ?BIG_REPEATS), + put_payload(small_chunk, ?SMALL_LINE), + put_payload(big_chunk, BigLine), elli_bench_sup:start_link(). stop(_State) -> ok. + +put_payload(Key, Value) -> + persistent_term:put({?MODULE, Key}, Value). + +payload(Key) -> + persistent_term:get({?MODULE, Key}). diff --git a/benchmark/elli/src/elli_bench_callback.erl b/benchmark/elli/src/elli_bench_callback.erl index 2905cc3..299f33b 100644 --- a/benchmark/elli/src/elli_bench_callback.erl +++ b/benchmark/elli/src/elli_bench_callback.erl @@ -6,11 +6,9 @@ -include_lib("elli/include/elli.hrl"). --define(SSE_EVENTS, 32). +-define(SMALL_COUNT, 100). +-define(BIG_COUNT, 64). -%% elli never sets TCP_NODELAY on accepted sockets, so multi write responses -%% (well chunked in particular) eat a Nagle and delayed-ACK stall between each -%% write. handle(Req, _Args) -> elli_tcp:setopts(Req#req.socket, [{nodelay, true}]), handle(Req#req.method, elli_request:path(Req), Req). @@ -18,8 +16,6 @@ handle(Req, _Args) -> handle('GET', [<<"hello">>], _Req) -> {ok, [], <<"Hello, Joe!">>}; -%% elli reads the full request body before calling the handler, so there is -%% no incremental and chunked read to distinguish from a plain echo. handle('POST', [<<"echo">>], Req) -> {ok, [], elli_request:body(Req)}; @@ -28,24 +24,37 @@ handle('GET', [<<"stream">>], Req) -> spawn(fun() -> stream_hello(Ref) end), {chunk, []}; -%% elli has no SSE API, so the stream is plain chunked writes carrying the -%% event framing. -handle('GET', [<<"sse">>], Req) -> - Ref = elli_request:chunk_ref(Req), - spawn(fun() -> start_events(Ref) end), - {chunk, [{<<"Content-Type">>, <<"text/event-stream">>}, - {<<"Cache-Control">>, <<"no-cache">>}]}; +handle('GET', [<<"stream">>, <<"small">>], Req) -> + burst(Req, elli_bench_app:payload(small_chunk), ?SMALL_COUNT); +handle('GET', [<<"stream">>, <<"big">>], Req) -> + burst(Req, elli_bench_app:payload(big_chunk), ?BIG_COUNT); +handle('GET', [<<"file">>, <<"tiny">>], _Req) -> + {ok, [{<<"Content-Type">>, <<"application/octet-stream">>}], + {file, "../priv/file_1kb.bin"}}; handle('GET', [<<"file">>, <<"small">>], _Req) -> {ok, [{<<"Content-Type">>, <<"application/octet-stream">>}], {file, "../priv/file_100kb.bin"}}; handle('GET', [<<"file">>, <<"big">>], _Req) -> {ok, [{<<"Content-Type">>, <<"application/octet-stream">>}], - {file, "../priv/file_1gb.bin"}}; + {file, "../priv/file_5mb.bin"}}; handle(_Method, _Path, _Req) -> {404, [], <<>>}. +burst(Req, Chunk, Count) -> + Ref = elli_request:chunk_ref(Req), + spawn(fun() -> send_burst(Ref, Chunk, Count) end), + {chunk, []}. + +send_burst(Ref, _Chunk, 0) -> + elli_request:close_chunk(Ref); +send_burst(Ref, Chunk, Remaining) -> + case elli_request:send_chunk(Ref, Chunk) of + ok -> send_burst(Ref, Chunk, Remaining - 1); + {error, _Reason} -> ok + end. + stream_hello(Ref) -> case elli_request:send_chunk(Ref, <<"hello, ">>) of ok -> @@ -56,28 +65,5 @@ stream_hello(Ref) -> {error, _reason} -> ok end. -start_events(Ref) -> - self() ! {sse_tick, 1}, - stream_events(Ref). - -stream_events(Ref) -> - receive - {sse_tick, N} when N > ?SSE_EVENTS -> - elli_request:close_chunk(Ref); - {sse_tick, N} -> - case elli_request:send_chunk(Ref, sse_event(N)) of - ok -> - self() ! {sse_tick, N + 1}, - stream_events(Ref); - {error, _Reason} -> - ok - end - end. - -sse_event(N) -> - Id = integer_to_binary(N), - <<"event: tick\nid: ", Id/binary, - "\ndata: {\"n\":", Id/binary, ",\"at\":\"benchmark\"}\n\n">>. - handle_event(_Event, _Args, _Config) -> ok. diff --git a/benchmark/ewe@4/src/app.gleam b/benchmark/ewe@4/src/app.gleam index 6b5a30b..cdf172e 100644 --- a/benchmark/ewe@4/src/app.gleam +++ b/benchmark/ewe@4/src/app.gleam @@ -5,7 +5,6 @@ import gleam/erlang/process.{type Subject} import gleam/http import gleam/http/request import gleam/http/response -import gleam/int import gleam/option import gleam/string import logging @@ -14,15 +13,42 @@ pub fn main() -> Nil { logging.configure() logging.set_level(logging.Debug) + let payload = payload() + let assert Ok(_started) = - ewe.new(handle_request) + ewe.new(handle_request(payload, _)) |> ewe.listening(port: 3001) |> ewe.start process.sleep_forever() } +const sse_events = 32 + +const small_count = 100 + +const big_count = 64 + +const big_repeats = 256 + +const small_line = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +type Payload { + Payload(small: BitArray, big: BitArray, big_line: String) +} + +fn payload() -> Payload { + let big_line = string.repeat(small_line, big_repeats) + + Payload( + small: bit_array.from_string(small_line), + big: bit_array.from_string(big_line), + big_line:, + ) +} + fn handle_request( + payload: Payload, request: request.Request(ewe.Connection), ) -> response.Response(ewe.ResponseBody) { case request.method, request.path { @@ -38,43 +64,31 @@ fn handle_request( } http.Post, "/echo/chunked" -> echo_chunked(request) http.Get, "/stream" -> stream_hello(request) - http.Get, "/sse" -> sse_burst(request) - http.Get, "/file/small" -> { - // head -c 100K /dev/urandom > file_100kb.bin - let assert Ok(file) = - ewe.file( - "../priv/file_100kb.bin", - offset: option.None, - limit: option.None, - ) - - response.Response( - status: 200, - headers: [#("content-type", "application/octet-stream")], - body: file, - ) - } - http.Get, "/file/big" -> { - // head -c 1G /dev/urandom > file_1gb.bin - let assert Ok(file) = - ewe.file( - "../priv/file_1gb.bin", - offset: option.None, - limit: option.None, - ) - - response.Response( - status: 200, - headers: [#("content-type", "application/octet-stream")], - body: file, - ) - } + http.Get, "/stream/small" -> + stream_burst(request, payload.small, small_count) + http.Get, "/stream/big" -> stream_burst(request, payload.big, big_count) + http.Get, "/sse" -> sse_burst(request, small_line, sse_events) + http.Get, "/sse/small" -> sse_burst(request, small_line, small_count) + http.Get, "/sse/big" -> sse_burst(request, payload.big_line, big_count) + http.Get, "/file/tiny" -> file("../priv/file_1kb.bin") + http.Get, "/file/small" -> file("../priv/file_100kb.bin") + http.Get, "/file/big" -> file("../priv/file_5mb.bin") _method, _path -> response.new(404) |> response.set_body(ewe.Empty) } } +fn file(path: String) -> response.Response(ewe.ResponseBody) { + let assert Ok(file) = ewe.file(path, offset: option.None, limit: option.None) + + response.Response( + status: 200, + headers: [#("content-type", "application/octet-stream")], + body: file, + ) +} + fn echo_chunked( request: request.Request(ewe.Connection), ) -> response.Response(ewe.ResponseBody) { @@ -121,15 +135,11 @@ fn stream_hello( }, handler: fn(conn, state, message) { case message { - StreamChunk(data) -> { + StreamChunk(data) -> case ewe.send_chunk(conn, data) { Ok(Nil) -> ewe.chunked_continue(state) - Error(reason) -> - ewe.chunked_stop_abnormal( - "Failed to send chunk: " <> string.inspect(reason), - ) + Error(_reason) -> ewe.chunked_stop_abnormal("failed to send chunk") } - } StreamDone -> ewe.chunked_stop() } }, @@ -137,14 +147,42 @@ fn stream_hello( ) } -const sse_events = 32 - type Tick { Tick(Int) } +fn stream_burst( + request: request.Request(ewe.Connection), + chunk: BitArray, + count: Int, +) -> response.Response(ewe.ResponseBody) { + ewe.chunked_body( + request, + response.new(200), + on_init: fn(subject: Subject(Tick)) { + process.send(subject, Tick(1)) + subject + }, + handler: fn(conn, subject, message) { + let Tick(n) = message + + case ewe.send_chunk(conn, chunk) { + Error(_reason) -> ewe.chunked_stop_abnormal("failed to send chunk") + Ok(Nil) if n >= count -> ewe.chunked_stop() + Ok(Nil) -> { + process.send(subject, Tick(n + 1)) + ewe.chunked_continue(subject) + } + } + }, + on_close: fn(_conn, _state) { Nil }, + ) +} + fn sse_burst( request: request.Request(ewe.Connection), + data: String, + count: Int, ) -> response.Response(ewe.ResponseBody) { ewe.sse( request, @@ -155,9 +193,9 @@ fn sse_burst( handler: fn(conn, subject, message) { let Tick(n) = message - case ewe.send_event(conn, tick_event(n)) { + case ewe.send_event(conn, ewe.event(data) |> ewe.event_name("tick")) { Error(_reason) -> ewe.sse_stop_abnormal("failed to send event") - Ok(Nil) if n >= sse_events -> ewe.sse_stop() + Ok(Nil) if n >= count -> ewe.sse_stop() Ok(Nil) -> { process.send(subject, Tick(n + 1)) ewe.sse_continue(subject) @@ -167,11 +205,3 @@ fn sse_burst( on_close: fn(_conn, _state) { Nil }, ) } - -fn tick_event(n: Int) -> ewe.SSEEvent { - let n = int.to_string(n) - - ewe.event("{\"n\":" <> n <> ",\"at\":\"benchmark\"}") - |> ewe.event_name("tick") - |> ewe.event_id(n) -} diff --git a/benchmark/ewe@5/manifest.toml b/benchmark/ewe@5/manifest.toml index ec6f173..ed622e4 100644 --- a/benchmark/ewe@5/manifest.toml +++ b/benchmark/ewe@5/manifest.toml @@ -7,16 +7,17 @@ # You should check this file into your source control repository. packages = [ - { name = "ewe", version = "5.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "glisten", "logging", "websocks"], source = "local", path = "../.." }, + { name = "alpacki", version = "3.0.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "alpacki", source = "hex", outer_checksum = "5BCB617A9606E56018790D4F24AB1E06B6D05BC17ACD6686B99E8D4D18B1F5FD" }, + { name = "ewe", version = "5.0.0", build_tools = ["gleam"], requirements = ["alpacki", "gleam_erlang", "gleam_http", "gleam_otp", "gleam_stdlib", "glisten", "logging", "websocks"], source = "local", path = "../.." }, { name = "gleam_crypto", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "2DE9E4EF53CF6FEE049D4F765731F7178F7A11AEFAE00EEE63BF7536B354AD3F" }, { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, { name = "gleam_http", version = "4.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "82EA6A717C842456188C190AFB372665EA56CE13D8559BF3B1DD9E40F619EE0C" }, - { name = "gleam_otp", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "BA6A294E295E428EC1562DC1C11EA7530DCB981E8359134BEABC8493B7B2258E" }, - { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, + { name = "gleam_otp", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_stdlib"], otp_app = "gleam_otp", source = "hex", outer_checksum = "DE4CA6850842F0266EE95317A25DD6A0A0F20CDFAB7C0ADC2E63251D7C3C72EC" }, + { name = "gleam_stdlib", version = "1.0.5", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "CEE5B6C076A85B45F60C585F4316C63EC8B7127C119D5738C3958A9C4D50404E" }, { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, - { name = "glisten", version = "9.0.1", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib", "logging"], source = "git", repo = "https://github.com/vshakitskiy/glisten.git", commit = "63e9a39f7dc35526c2f62128f44f871dab245d15" }, + { name = "glisten", version = "9.0.1", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_otp", "gleam_stdlib", "logging"], source = "git", repo = "https://github.com/vshakitskiy/glisten.git", commit = "81a6b0005451b61a14dad2f5892d04cbaad3bc5d" }, { name = "logging", version = "1.5.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "logging", source = "hex", outer_checksum = "BC5F18CE5DD9686100229FE5409BDC3DD5C46D5A7DF2F804AD2D8F0DD6C5060E" }, - { name = "websocks", version = "3.0.1", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_erlang", "gleam_stdlib"], otp_app = "websocks", source = "hex", outer_checksum = "C70340E5B6C3390383ADA17029DCA6F8903863A7AD8CD8E1520EDCC4FE70D6FD" }, + { name = "websocks", version = "4.0.1", build_tools = ["gleam"], requirements = ["gleam_crypto", "gleam_erlang", "gleam_stdlib"], otp_app = "websocks", source = "hex", outer_checksum = "89B0C31A032CBE28D4C5FB5CC25A8A690669FEBA501C63F99A4E8F5748750B2A" }, ] [requirements] diff --git a/benchmark/ewe@5/src/app.gleam b/benchmark/ewe@5/src/app.gleam index fece45b..a35aee3 100644 --- a/benchmark/ewe@5/src/app.gleam +++ b/benchmark/ewe@5/src/app.gleam @@ -1,11 +1,13 @@ import ewe +import gleam/bit_array import gleam/bytes_tree import gleam/erlang/process import gleam/http import gleam/http/request import gleam/http/response -import gleam/int import gleam/option +import gleam/result +import gleam/string import logging pub fn main() -> Nil { @@ -14,16 +16,43 @@ pub fn main() -> Nil { let listener_name = process.new_name("listener_name") let connection_factory_name = process.new_name("connection_factory_name") + let payload = payload() + let handler = handle_request(payload, _) let assert Ok(_started) = - ewe.new(listener_name:, connection_factory_name:, handler: handle_request) + ewe.new(listener_name:, connection_factory_name:, handler:) |> ewe.listening(on: 3006) |> ewe.start process.sleep_forever() } +const sse_events = 32 + +const small_count = 100 + +const big_count = 64 + +const big_repeats = 256 + +const small_line = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +type Payload { + Payload(small: BitArray, big: BitArray, big_line: String) +} + +fn payload() -> Payload { + let big_line = string.repeat(small_line, big_repeats) + + Payload( + small: bit_array.from_string(small_line), + big: bit_array.from_string(big_line), + big_line:, + ) +} + fn handle_request( + payload: Payload, request: request.Request(ewe.Connection), ) -> response.Response(ewe.Body) { case request.method, request.path { @@ -40,48 +69,37 @@ fn handle_request( http.Post, "/echo/chunked" -> echo_chunked(request, bytes_tree.new()) http.Get, "/stream" -> { use writer <- ewe.stream_response(response.new(200)) - let writer = ewe.send_chunk(writer, <<"hello, ":utf8>>) + use writer <- result.try(ewe.send_chunk(writer, <<"hello, ":utf8>>)) ewe.finish_chunk(writer, <<"Joe!":utf8>>) } - http.Get, "/sse" -> sse_burst() - http.Get, "/file/small" -> { - // head -c 100K /dev/urandom > file_100kb.bin - let assert Ok(file) = - ewe.file( - request.body, - "../priv/file_100kb.bin", - offset: option.None, - limit: option.None, - ) - - response.Response( - status: 200, - headers: [#("content-type", "application/octet-stream")], - body: file, - ) - } - http.Get, "/file/big" -> { - // head -c 1G /dev/urandom > file_1gb.bin - let assert Ok(file) = - ewe.file( - request.body, - "../priv/file_1gb.bin", - offset: option.None, - limit: option.None, - ) - - response.Response( - status: 200, - headers: [#("content-type", "application/octet-stream")], - body: file, - ) - } + http.Get, "/stream/small" -> stream_burst(payload.small, small_count) + http.Get, "/stream/big" -> stream_burst(payload.big, big_count) + http.Get, "/sse" -> sse_burst(small_line, sse_events) + http.Get, "/sse/small" -> sse_burst(small_line, small_count) + http.Get, "/sse/big" -> sse_burst(payload.big_line, big_count) + http.Get, "/file/tiny" -> file(request, "../priv/file_1kb.bin") + http.Get, "/file/small" -> file(request, "../priv/file_100kb.bin") + http.Get, "/file/big" -> file(request, "../priv/file_5mb.bin") _method, _path -> response.new(404) |> response.set_body(ewe.Empty) } } +fn file( + request: request.Request(ewe.Connection), + path: String, +) -> response.Response(ewe.Body) { + let assert Ok(file) = + ewe.file(request.body, path, offset: option.None, limit: option.None) + + response.Response( + status: 200, + headers: [#("content-type", "application/octet-stream")], + body: file, + ) +} + fn echo_chunked( request: request.Request(ewe.Connection), acc: bytes_tree.BytesTree, @@ -95,13 +113,30 @@ fn echo_chunked( } } -const sse_events = 32 +fn stream_burst(chunk: BitArray, count: Int) -> response.Response(ewe.Body) { + use writer <- ewe.stream_response(response.new(200)) + stream_chunks(writer, chunk, count) +} + +fn stream_chunks( + writer: ewe.ResponseWriter, + chunk: BitArray, + remaining: Int, +) -> Result(Nil, ewe.SendError) { + case remaining { + 1 -> ewe.finish_chunk(writer, chunk) + _remaining -> { + use writer <- result.try(ewe.send_chunk(writer, chunk)) + stream_chunks(writer, chunk, remaining - 1) + } + } +} type Tick { Tick(Int) } -fn sse_burst() -> response.Response(ewe.Body) { +fn sse_burst(data: String, count: Int) -> response.Response(ewe.Body) { ewe.sse( response.new(200), on_init: fn(subject) { @@ -111,9 +146,9 @@ fn sse_burst() -> response.Response(ewe.Body) { handler: fn(conn, subject, message) { let Tick(n) = message - case ewe.send_event(conn, tick_event(n)) { + case ewe.send_event(conn, ewe.event(data) |> ewe.event_name("tick")) { Error(_reason) -> ewe.sse_stop_abnormal("failed to send event") - Ok(Nil) if n >= sse_events -> ewe.sse_stop() + Ok(Nil) if n >= count -> ewe.sse_stop() Ok(Nil) -> { process.send(subject, Tick(n + 1)) ewe.sse_continue(subject) @@ -123,11 +158,3 @@ fn sse_burst() -> response.Response(ewe.Body) { on_close: fn(_conn, _state) { Nil }, ) } - -fn tick_event(n: Int) -> ewe.SseEvent { - let n = int.to_string(n) - - ewe.event("{\"n\":" <> n <> ",\"at\":\"benchmark\"}") - |> ewe.event_name("tick") - |> ewe.event_id(n) -} diff --git a/benchmark/httpd/src/httpd_bench_app.erl b/benchmark/httpd/src/httpd_bench_app.erl index 533525c..1624e2c 100644 --- a/benchmark/httpd/src/httpd_bench_app.erl +++ b/benchmark/httpd/src/httpd_bench_app.erl @@ -2,10 +2,22 @@ -behaviour(application). --export([start/2, stop/1]). +-export([start/2, stop/1, payload/1]). + +-define(SMALL_LINE, <<"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef">>). +-define(BIG_REPEATS, 256). start(_StartType, _StartArgs) -> + BigLine = binary:copy(?SMALL_LINE, ?BIG_REPEATS), + put_payload(small_chunk, ?SMALL_LINE), + put_payload(big_chunk, BigLine), httpd_bench_sup:start_link(). stop(_State) -> ok. + +put_payload(Key, Value) -> + persistent_term:put({?MODULE, Key}, Value). + +payload(Key) -> + persistent_term:get({?MODULE, Key}). diff --git a/benchmark/httpd/src/httpd_bench_callback.erl b/benchmark/httpd/src/httpd_bench_callback.erl index 8b0ae99..cb09cf9 100644 --- a/benchmark/httpd/src/httpd_bench_callback.erl +++ b/benchmark/httpd/src/httpd_bench_callback.erl @@ -7,11 +7,9 @@ -define(FILE_CHUNK_SIZE, 262144). --define(SSE_EVENTS, 32). +-define(SMALL_COUNT, 100). +-define(BIG_COUNT, 64). -%% inets never sets TCP_NODELAY on accepted sockets and exposes no config for it, -%% so every keep-alive response after the first eats a ~40ms Nagle and -%% delayed-ACK stall. do(Info) -> #mod{socket_type = SocketType, socket = Socket} = Info, ensure_nodelay(SocketType, Socket), @@ -25,8 +23,6 @@ ensure_nodelay(_SocketType, _Socket) -> route("GET", "/hello", _Info) -> {break, [{response, {200, "Hello, Joe!"}}]}; -%% httpd reads the full request body before calling the module, so there is -%% no incremental and chunked read to distinguish from a plain echo. route("POST", "/echo", Info) -> {break, [{response, {200, entity_body(Info)}}]}; @@ -34,16 +30,19 @@ route("GET", "/stream", Info) -> send_stream(Info), {break, [{response, {already_sent, 200, 0}}]}; -%% httpd has no SSE API, so the stream is written straight to the socket. -route("GET", "/sse", Info) -> - send_sse(Info), - {break, [{response, {already_sent, 200, 0}}]}; +route("GET", "/stream/small", Info) -> + send_burst(Info, "text/plain", small_chunk, ?SMALL_COUNT); +route("GET", "/stream/big", Info) -> + send_burst(Info, "application/octet-stream", big_chunk, ?BIG_COUNT); +route("GET", "/file/tiny", Info) -> + send_file(Info, "../priv/file_1kb.bin"), + {break, [{response, {already_sent, 200, 0}}]}; route("GET", "/file/small", Info) -> send_file(Info, "../priv/file_100kb.bin"), {break, [{response, {already_sent, 200, 0}}]}; route("GET", "/file/big", Info) -> - send_file(Info, "../priv/file_1gb.bin"), + send_file(Info, "../priv/file_5mb.bin"), {break, [{response, {already_sent, 200, 0}}]}; route(_Method, _Uri, Info) -> @@ -55,53 +54,38 @@ entity_body(Info) -> Body -> Body end. -send_stream(Info) -> - #mod{socket_type = SocketType, socket = Socket} = Info, - Head = +chunked_head(ContentType) -> + [ "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" + "Content-Type: ", ContentType, "\r\n" "Transfer-Encoding: chunked\r\n" - "Connection: keep-alive\r\n\r\n", - httpd_socket:deliver(SocketType, Socket, Head), - send_chunk(SocketType, Socket, "hello, "), - send_chunk(SocketType, Socket, "Joe!"), - httpd_socket:deliver(SocketType, Socket, "0\r\n\r\n"). + "Connection: keep-alive\r\n\r\n" + ]. -send_sse(Info) -> +send_stream(Info) -> #mod{socket_type = SocketType, socket = Socket} = Info, - Head = - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/event-stream\r\n" - "Cache-Control: no-cache\r\n" - "Transfer-Encoding: chunked\r\n" - "Connection: keep-alive\r\n\r\n", - httpd_socket:deliver(SocketType, Socket, Head), - self() ! {sse_tick, 1}, - send_sse_events(SocketType, Socket), + httpd_socket:deliver(SocketType, Socket, chunked_head("text/plain")), + send_chunk(SocketType, Socket, <<"hello, ">>), + send_chunk(SocketType, Socket, <<"Joe!">>), httpd_socket:deliver(SocketType, Socket, "0\r\n\r\n"). -send_sse_events(SocketType, Socket) -> - receive - {sse_tick, N} when N > ?SSE_EVENTS -> - ok; - {sse_tick, N} -> - send_chunk(SocketType, Socket, sse_event(N)), - self() ! {sse_tick, N + 1}, - send_sse_events(SocketType, Socket) - end. +send_burst(Info, ContentType, Key, Count) -> + #mod{socket_type = SocketType, socket = Socket} = Info, + httpd_socket:deliver(SocketType, Socket, chunked_head(ContentType)), + deliver_burst(SocketType, Socket, httpd_bench_app:payload(Key), Count), + httpd_socket:deliver(SocketType, Socket, "0\r\n\r\n"), + {break, [{response, {already_sent, 200, 0}}]}. -sse_event(N) -> - Id = integer_to_list(N), - lists:flatten([ - "event: tick\nid: ", Id, - "\ndata: {\"n\":", Id, ",\"at\":\"benchmark\"}\n\n" - ]). +deliver_burst(_SocketType, _Socket, _Chunk, 0) -> + ok; +deliver_burst(SocketType, Socket, Chunk, Remaining) -> + send_chunk(SocketType, Socket, Chunk), + deliver_burst(SocketType, Socket, Chunk, Remaining - 1). send_chunk(SocketType, Socket, Data) -> - Size = integer_to_list(length(Data), 16), + Size = integer_to_list(byte_size(Data), 16), httpd_socket:deliver(SocketType, Socket, [Size, "\r\n", Data, "\r\n"]). -%% There is no built-in equivalent of the other server's sendfile helpers send_file(Info, Path) -> #mod{socket_type = SocketType, socket = Socket} = Info, {ok, #file_info{size = Size}} = file:read_file_info(Path), diff --git a/benchmark/latency.sh b/benchmark/latency.sh new file mode 100755 index 0000000..fe63ad5 --- /dev/null +++ b/benchmark/latency.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Usage: ./latency.sh [--servers ewe@5,bandit] [--cases hello,sse_big] + +set -u + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT" +source "$ROOT/config.sh" +source "$ROOT/lib.sh" + +ONLY_SERVERS="" +ONLY_CASES="" +while [ $# -gt 0 ]; do + case "$1" in + --servers) ONLY_SERVERS="$2"; shift 2 ;; + --cases) ONLY_CASES="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +WRK2="${WRK2:-$ROOT/.wrk2/wrk2}" +[ -x "$WRK2" ] || WRK2="$(command -v wrk2 || true)" +[ -x "$WRK2" ] || { echo "wrk2 not found, run ./wrk2-setup.sh first" >&2; exit 1; } + +ensure_fixtures +new_results_dir latency +CSV="$RESULTS_DIR/latency.csv" +echo "server,case,target_rate,achieved_rate,p50,p90,p99,p999,p50_raw,p99_raw,errors" > "$CSV" + +{ + echo "date $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "load wrk2 http/1, ${CONNECTIONS} connections, ${THREADS} threads, fixed rates" + echo "duration ${DURATION}s per rate" + echo "cpu server on ${SERVER_CPUS}, load generator on ${LOAD_CPUS}" +} | tee "$RESULTS_DIR/run.txt" +echo + +parse_wrk2() { + awk ' + /Latency Distribution \(HdrHistogram - Recorded Latency\)/ { section = "corrected"; next } + /Latency Distribution \(HdrHistogram - Uncorrected Latency/ { section = "raw"; next } + section == "corrected" && / 50\.000%/ { p50 = $2 } + section == "corrected" && / 90\.000%/ { p90 = $2 } + section == "corrected" && / 99\.000%/ { p99 = $2 } + section == "corrected" && / 99\.900%/ { p999 = $2 } + section == "raw" && / 50\.000%/ { raw50 = $2 } + section == "raw" && / 99\.000%/ { raw99 = $2 } + /Requests\/sec:/ { rate = $2; finished = 1 } + /Socket errors:/ { gsub(",", ""); errors = $0 } + END { + if (!finished) exit 1 + printf "%s|%s|%s|%s|%s|%s|%s|%s\n", rate, p50, p90, p99, p999, raw50, raw99, errors + } + ' +} + +run_rate() { + local rate="$1" script raw parsed + local achieved p50 p90 p99 p999 raw50 raw99 errors + local args=() + + script="$(wrk_script "$case_name")" + [ -n "$script" ] && args+=(-s "$ROOT/wrk/$script") + + raw="$RESULTS_DIR/${server}__${case_name}__${rate}.txt" + if ! parsed="$("${PIN_LOAD[@]}" "$WRK2" -t"$THREADS" -c"$CONNECTIONS" -d"${DURATION}s" \ + -R"$rate" -U "${args[@]}" "http://127.0.0.1:$port$path" 2>&1 | tee "$raw" | parse_wrk2)"; then + printf ' %8s target failed, see %s\n' "$rate" "$raw" + return 1 + fi + + IFS='|' read -r achieved p50 p90 p99 p999 raw50 raw99 errors <<< "$parsed" + + echo "$server,$case_name,$rate,$achieved,$p50,$p90,$p99,$p999,$raw50,$raw99,$errors" >> "$CSV" + printf ' %8s target %10s actual p50 %9s p99 %9s raw p99 %9s\n' \ + "$rate" "$achieved" "$p50" "$p99" "$raw99" +} + +for server_entry in "${SERVERS[@]}"; do + IFS='|' read -r server dir cmd h1_port _h2_port <<< "$server_entry" + selected "$server" "$ONLY_SERVERS" || continue + [ "$h1_port" = "-" ] && continue + port="$h1_port" + + echo "=== $server ===" + start_server "$dir" "$cmd" "$port" "$RESULTS_DIR/$server.log" || continue + + "${PIN_LOAD[@]}" "$WRK2" -t"$THREADS" -c"$CONNECTIONS" -d"${WARMUP}s" -R"$WARMUP_RATE" \ + "http://127.0.0.1:$port/hello" > /dev/null 2>&1 + + for case_entry in "${CASES[@]}"; do + IFS='|' read -r case_name path _body _headers _messages <<< "$case_entry" + selected "$case_name" "$ONLY_CASES" || continue + + if unsupported "$server" "$case_name"; then + printf ' %-20s not supported\n' "$case_name" + continue + fi + + echo " $case_name" + for rate in $(latency_rates "$case_name"); do + run_rate "$rate" + done + done + + stop_server + echo +done + +render_report "$CSV" diff --git a/benchmark/lib.sh b/benchmark/lib.sh new file mode 100644 index 0000000..21968af --- /dev/null +++ b/benchmark/lib.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +CORES="$(nproc)" +if [ "$CORES" -ge 4 ]; then + SERVER_CPUS="0-$((CORES / 2 - 1))" + LOAD_CPUS="$((CORES / 2))-$((CORES - 1))" + PIN_SERVER=(taskset -c "$SERVER_CPUS") + PIN_LOAD=(taskset -c "$LOAD_CPUS") +else + SERVER_CPUS="unpinned" + LOAD_CPUS="unpinned" + PIN_SERVER=() + PIN_LOAD=() +fi + +fixture() { + local name="$1" size="$2" + [ -f "$ROOT/priv/$name" ] || head -c "$size" /dev/urandom > "$ROOT/priv/$name" +} + +ensure_fixtures() { + mkdir -p "$ROOT/priv" + fixture file_1kb.bin 1K + fixture file_100kb.bin 100K + fixture file_5mb.bin 5M + fixture body_1kb.bin 1K + fixture body_10kb.bin 10K +} + +body_file() { + case "$1" in + -) echo "" ;; + *) echo "$ROOT/priv/body_$1.bin" ;; + esac +} + +new_results_dir() { + RESULTS_DIR="$ROOT/results/$(date +%Y%m%d-%H%M%S)-$1" + mkdir -p "$RESULTS_DIR" +} + +render_report() { + local csv="$1" + echo "Results: $csv" + command -v deno > /dev/null || return 0 + echo + "$ROOT/report.js" "$csv" | tee "$RESULTS_DIR/report.txt" + echo "Report: $RESULTS_DIR/report.txt" +} + +port_pid() { + ss -tlnp 2>/dev/null | awk -v p=":$1\$" '$4 ~ p' | grep -oP 'pid=\K[0-9]+' | head -1 +} + +pgid_of() { + ps -o pgid= -p "$1" 2>/dev/null | tr -d ' ' +} + +free_port() { + local port="$1" pid attempts=5 + + [ "$port" = "-" ] && return 0 + while pid="$(port_pid "$port")" && [ -n "$pid" ]; do + if [ "$attempts" -eq 0 ]; then + echo " port $port still held by pid $pid, giving up" >&2 + return 1 + fi + echo " port $port held by pid $pid, killing" >&2 + kill -9 "$pid" 2>/dev/null + attempts=$((attempts - 1)) + sleep 1 + done +} + +port_answered_by_group() { + local port="$1" pgid="$2" pid + pid="$(port_pid "$port")" + [ -z "$pid" ] && return 0 + [ "$(pgid_of "$pid")" = "$pgid" ] +} + +startup_port() { + if [ "$1" != "-" ]; then echo "$1"; else echo "$2"; fi +} + +wait_for_port() { + local port="$1" attempts=60 + while [ "$attempts" -gt 0 ]; do + (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null && { exec 3<&- 3>&-; return 0; } + sleep 0.5 + attempts=$((attempts - 1)) + done + return 1 +} + +CURRENT_PGID="" + +start_server() { + local dir="$1" cmd="$2" port="$3" log="$4" + + free_port "$port" || return 1 + + ( cd "$ROOT/$dir" && exec setsid "${PIN_SERVER[@]}" $cmd > "$log" 2>&1 < /dev/null ) & + CURRENT_PGID=$! + + if ! wait_for_port "$port"; then + echo " failed to start, see $log" >&2 + stop_server + return 1 + fi + + if ! port_answered_by_group "$port" "$CURRENT_PGID"; then + echo " port $port is answered by another process, see $log" >&2 + stop_server + return 1 + fi +} + +stop_server() { + [ -z "$CURRENT_PGID" ] && return 0 + kill -TERM -- -"$CURRENT_PGID" 2>/dev/null + sleep 1 + kill -KILL -- -"$CURRENT_PGID" 2>/dev/null + wait "$CURRENT_PGID" 2>/dev/null + CURRENT_PGID="" +} + +trap stop_server EXIT INT TERM HUP PIPE + +selected() { + local name="$1" allowed="$2" + [ -z "$allowed" ] && return 0 + case ",$allowed," in + *",$name,"*) return 0 ;; + *) return 1 ;; + esac +} diff --git a/benchmark/mist/src/app.gleam b/benchmark/mist/src/app.gleam index fcb01e7..96b67d3 100644 --- a/benchmark/mist/src/app.gleam +++ b/benchmark/mist/src/app.gleam @@ -4,9 +4,9 @@ import gleam/erlang/process.{type Subject} import gleam/http import gleam/http/request import gleam/http/response -import gleam/int import gleam/option import gleam/otp/actor +import gleam/string import gleam/string_tree import logging import mist @@ -15,15 +15,42 @@ pub fn main() -> Nil { logging.configure() logging.set_level(logging.Debug) + let payload = payload() + let assert Ok(_started) = - mist.new(handle_request) + mist.new(handle_request(payload, _)) |> mist.port(3002) |> mist.start process.sleep_forever() } +const sse_events = 32 + +const small_count = 100 + +const big_count = 64 + +const big_repeats = 256 + +const small_line = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +type Payload { + Payload(small: BitArray, big: BitArray, big_line: String) +} + +fn payload() -> Payload { + let big_line = string.repeat(small_line, big_repeats) + + Payload( + small: bit_array.from_string(small_line), + big: bit_array.from_string(big_line), + big_line:, + ) +} + fn handle_request( + payload: Payload, request: request.Request(mist.Connection), ) -> response.Response(mist.ResponseData) { case request.method, request.path { @@ -40,35 +67,31 @@ fn handle_request( } http.Post, "/echo/chunked" -> echo_chunked(request) http.Get, "/stream" -> stream_hello(request) - http.Get, "/sse" -> sse_burst(request) - http.Get, "/file/small" -> { - // head -c 100K /dev/urandom > file_100kb.bin - let assert Ok(file) = - mist.send_file("../priv/file_100kb.bin", offset: 0, limit: option.None) - - response.Response( - status: 200, - headers: [#("content-type", "application/octet-stream")], - body: file, - ) - } - http.Get, "/file/big" -> { - // head -c 1G /dev/urandom > file_1gb.bin - let assert Ok(file) = - mist.send_file("../priv/file_1gb.bin", offset: 0, limit: option.None) - - response.Response( - status: 200, - headers: [#("content-type", "application/octet-stream")], - body: file, - ) - } + http.Get, "/stream/small" -> + stream_burst(request, payload.small, small_count) + http.Get, "/stream/big" -> stream_burst(request, payload.big, big_count) + http.Get, "/sse" -> sse_burst(request, small_line, sse_events) + http.Get, "/sse/small" -> sse_burst(request, small_line, small_count) + http.Get, "/sse/big" -> sse_burst(request, payload.big_line, big_count) + http.Get, "/file/tiny" -> file("../priv/file_1kb.bin") + http.Get, "/file/small" -> file("../priv/file_100kb.bin") + http.Get, "/file/big" -> file("../priv/file_5mb.bin") _method, _path -> response.new(404) |> response.set_body(mist.Bytes(bytes_tree.new())) } } +fn file(path: String) -> response.Response(mist.ResponseData) { + let assert Ok(file) = mist.send_file(path, offset: 0, limit: option.None) + + response.Response( + status: 200, + headers: [#("content-type", "application/octet-stream")], + body: file, + ) +} + fn echo_chunked( request: request.Request(mist.Connection), ) -> response.Response(mist.ResponseData) { @@ -126,18 +149,41 @@ fn stream_hello( ) } -/// Events emitted per `/sse` stream. Fixed across every benchmarked server so -/// that streams/sec times this is a comparable events/sec. -const sse_events = 32 - type Tick { Tick(Int) } -/// Emits `sse_events` events back to back with no pacing, then ends the -/// stream. A paced stream would measure the timer rather than the server. +fn stream_burst( + request: request.Request(mist.Connection), + chunk: BitArray, + count: Int, +) -> response.Response(mist.ResponseData) { + mist.chunked( + request:, + response: response.new(200), + init: fn(subject: Subject(Tick)) { + process.send(subject, Tick(1)) + subject + }, + loop: fn(subject, message, connection) { + let Tick(n) = message + + case mist.send_chunk(connection, chunk) { + Error(Nil) -> mist.chunk_stop_abnormal("failed to send chunk") + Ok(Nil) if n >= count -> mist.chunk_stop() + Ok(Nil) -> { + process.send(subject, Tick(n + 1)) + mist.chunk_continue(subject) + } + } + }, + ) +} + fn sse_burst( request: request.Request(mist.Connection), + data: String, + count: Int, ) -> response.Response(mist.ResponseData) { mist.server_sent_events( request:, @@ -149,9 +195,9 @@ fn sse_burst( loop: fn(subject, message, connection) { let Tick(n) = message - case mist.send_event(connection, tick_event(n)) { + case mist.send_event(connection, tick_event(data)) { Error(Nil) -> actor.stop() - Ok(Nil) if n >= sse_events -> actor.stop() + Ok(Nil) if n >= count -> actor.stop() Ok(Nil) -> { process.send(subject, Tick(n + 1)) actor.continue(subject) @@ -161,11 +207,8 @@ fn sse_burst( ) } -fn tick_event(n: Int) -> mist.SSEEvent { - let n = int.to_string(n) - - string_tree.from_string("{\"n\":" <> n <> ",\"at\":\"benchmark\"}") +fn tick_event(data: String) -> mist.SSEEvent { + string_tree.from_string(data) |> mist.event |> mist.event_name("tick") - |> mist.event_id(n) } diff --git a/benchmark/report.js b/benchmark/report.js new file mode 100755 index 0000000..fa301f5 --- /dev/null +++ b/benchmark/report.js @@ -0,0 +1,213 @@ +#!/usr/bin/env -S deno run --allow-read + +const COLUMN = 12; +const COLLAPSED = 20; +const NOISE = 0.05; + +function readCsv(path) { + const text = Deno.readTextFileSync(path); + const [header, ...lines] = text.trim().split("\n"); + const columns = header.split(","); + return lines.map((line) => { + const cells = line.split(","); + return Object.fromEntries(columns.map((name, i) => [name, cells[i]])); + }); +} + +function table(headings, widths, columns, rows) { + const head = headings.map((name, i) => name.padEnd(widths[i])).join(""); + console.log(head + columns.map((name) => name.padStart(COLUMN)).join("")); + console.log("-".repeat(head.length + COLUMN * columns.length)); + + for (const row of rows) { + const labels = row.labels.map((label, i) => label.padEnd(widths[i])).join(""); + const cells = columns.map((name) => (row.values.get(name) ?? "-").padStart(COLUMN)); + console.log(labels + cells.join("")); + } +} + +function section(title, lines) { + if (lines.length === 0) return; + console.log(`\n${title}\n`); + for (const line of lines) console.log(line); +} + +function groupBy(rows, keyOf) { + const groups = new Map(); + for (const row of rows) { + const key = keyOf(row); + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(row); + } + return groups; +} + +const unique = (values) => [...new Set(values)]; + +function median(numbers) { + const sorted = [...numbers].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +} + +function spread(numbers) { + const middle = median(numbers); + return middle ? (Math.max(...numbers) - Math.min(...numbers)) / middle : 0; +} + +const caseProfile = (row) => `${row.case}|${row.profile}`; +const cellName = (row) => `${row.server} ${row.profile} ${row.case}`; +const rateOf = (row) => Number(row.requests_per_sec); + +function throughputTable(rows, field) { + return [...groupBy(rows, caseProfile)].map(([key, group]) => { + const [name, profile] = key.split("|"); + const values = new Map( + [...groupBy(group, (row) => row.server)].map(([server, repeats]) => [ + server, + Math.round(median(repeats.map((row) => Number(row[field])))).toLocaleString(), + ]), + ); + return { labels: [name, profile], values }; + }); +} + +function throughput(rows) { + const servers = unique(rows.map((row) => row.server)).sort(); + const widths = [20, 11]; + + console.log("profiles"); + for (const [profile, group] of groupBy(rows, (row) => row.profile)) { + const { protocol, connections, streams } = group[0]; + console.log(` ${profile.padEnd(11)}${protocol}, ${connections} connections x ${streams} stream(s)`); + } + + console.log("\nthroughput req/s\n"); + table(["case", "profile"], widths, servers, throughputTable(rows, "requests_per_sec")); + + const streaming = rows.filter((row) => Number(row.messages) > 1); + if (streaming.length) { + console.log("\nmessages/s\n"); + table(["case", "profile"], widths, servers, throughputTable(streaming, "messages_per_sec")); + } + + warnings(rows); +} + +function warnings(rows) { + const noisy = []; + const broken = []; + const collapsed = []; + + const peers = new Map( + [...groupBy(rows, caseProfile)].map(([key, group]) => [key, median(group.map(rateOf))]), + ); + + for (const [name, group] of groupBy(rows, cellName)) { + const measured = group.map(rateOf); + const disagreement = spread(measured); + if (disagreement > NOISE) { + noisy.push(` ${name}: ${(disagreement * 100).toFixed(0)}% apart across repeats`); + } + + const failed = group.reduce((total, row) => total + Number(row.failed), 0); + const non2xx = group.reduce((total, row) => total + Number(row.non_2xx), 0); + if (failed || non2xx) broken.push(` ${name}: ${failed} failed, ${non2xx} non-2xx`); + + const peer = peers.get(caseProfile(group[0])); + const rate = median(measured); + if (peer && rate * COLLAPSED < peer) { + collapsed.push( + ` ${name}: ${Math.round(rate).toLocaleString()} req/s against a ` + + `${Math.round(peer).toLocaleString()} median for this case`, + ); + } + } + + section("not answering the case correctly", broken); + section("collapsed against peers", collapsed); + section("repeats disagreed by over 5%", noisy); +} + +function micros(value) { + const match = String(value).match(/^([\d.]+)(us|ms|s)?$/); + if (!match) return 0; + return Number(match[1]) * ({ us: 1, ms: 1000, s: 1000000 }[match[2]] ?? 1); +} + +function latencyOf(value) { + return micros(value) === 0 ? "-" : value; +} + +function hardestRates(rows) { + const byCaseServer = groupBy(rows, (row) => `${row.case}|${row.server}`); + return [...byCaseServer.values()].map((group) => + group.reduce((hardest, row) => + Number(row.target_rate) > Number(hardest.target_rate) ? row : hardest + ) + ); +} + +function latency(rows) { + const servers = unique(rows.map((row) => row.server)).sort(); + + console.log("p99 at a fixed offered rate for http1\n"); + for (const [name, forCase] of groupBy(rows, (row) => row.case)) { + console.log(name); + table( + ["target req/s"], + [14], + servers, + [...groupBy(forCase, (row) => row.target_rate)].map(([rate, group]) => ({ + labels: [Number(rate).toLocaleString()], + values: new Map(group.map((row) => [row.server, latencyOf(row.p99)])), + })), + ); + console.log(); + } + + console.log("CO gap at the highest rate\n"); + table( + ["case"], + [20], + servers, + [...groupBy(hardestRates(rows), (row) => row.case)].map(([name, group]) => ({ + labels: [name], + values: new Map( + group.map((row) => { + const raw = micros(row.p99_raw); + return [row.server, raw ? `${(micros(row.p99) / raw).toFixed(1)}x` : "-"]; + }), + ), + })), + ); +} + +function main() { + const paths = Deno.args; + if (paths.length === 0) { + console.error("usage: ./report.js ..."); + Deno.exit(1); + } + + for (const path of paths) { + let rows; + try { + rows = readCsv(path); + } catch { + console.error(`cannot read ${path}`); + Deno.exit(1); + } + + if (rows.length === 0) { + console.error(`no results in ${path}`); + Deno.exit(1); + } + + console.log(`${path}\n`); + if ("protocol" in rows[0]) throughput(rows); + else latency(rows); + console.log(); + } +} + +main(); diff --git a/benchmark/results.txt b/benchmark/results.txt new file mode 100644 index 0000000..2e79bba --- /dev/null +++ b/benchmark/results.txt @@ -0,0 +1,224 @@ +results/20260812-115143-throughput/throughput.csv + +profiles + h1 h1, 50 connections x 1 stream(s) + h2 h2, 50 connections x 10 stream(s) + +throughput req/s + +case profile bandit chatterbox elli ewe@4 ewe@5 httpd mist roadrunner +------------------------------------------------------------------------------------------------------------------------------- +hello h1 143,952 - 254,313 179,110 280,317 105,048 198,161 250,777 +hello_headers h1 120,941 - 211,013 139,815 220,993 87,467 154,745 218,903 +echo_1kb h1 132,502 - 230,092 156,755 221,848 71,946 172,462 226,440 +echo_1kb_headers h1 112,300 - 202,972 126,766 177,464 56,481 140,312 199,269 +echo_10kb h1 114,119 - 167,679 143,155 189,310 12,695 155,060 187,493 +echo_chunked_10kb h1 103,110 - - 136,347 194,886 - 151,108 184,883 +file_tiny h1 48,733 - 40,163 5 68,701 32,584 38,134 39,110 +file_small h1 45,200 - 37,416 5 58,562 28,876 35,437 36,329 +file_big h1 3,395 - 3,420 5 3,444 1,427 3,366 3,420 +stream h1 81,252 - 5 5 139,122 99,882 6 33,801 +stream_small h1 6,956 - 5 5 7,819 7,920 5 6,540 +stream_big h1 6,639 - 5 5 7,843 7,711 5 7,408 +sse h1 - - - 0 17,391 - 0 12,064 +sse_small h1 - - - 0 6,073 - 0 5,789 +sse_big h1 - - - 0 6,183 - 0 2,416 +hello h2 77,640 8,263 - - 166,562 - - 216,123 +hello_headers h2 70,243 8,194 - - 135,829 - - 171,430 +echo_1kb h2 65,957 7,234 - - 116,103 - - 184,712 +echo_1kb_headers h2 61,621 7,166 - - 101,558 - - 148,742 +echo_10kb h2 56,528 8,655 - - 95,871 - - 141,593 +echo_chunked_10kb h2 53,294 8,661 - - 92,825 - - 136,347 +file_tiny h2 38,528 8,311 - - 81,024 - - 35,231 +file_small h2 31,128 26,094 - - 51,984 - - 26,896 +file_big h2 750 553 - - 454 - - 989 +stream h2 49,502 8,721 - - 90,205 - - 119,662 +stream_small h2 4,241 10,781 - - 5,306 - - 6,479 +stream_big h2 4,389 799 - - 5,395 - - 6,332 +sse h2 - - - - 10,768 - - 16,107 +sse_small h2 - - - - 3,723 - - 5,622 +sse_big h2 - - - - 3,390 - - 2,166 + +messages/s + +case profile bandit chatterbox elli ewe@4 ewe@5 httpd mist roadrunner +------------------------------------------------------------------------------------------------------------------------------- +stream h1 162,504 - 10 10 278,244 199,764 12 67,602 +stream_small h1 695,610 - 500 500 781,910 791,990 500 654,020 +stream_big h1 424,896 - 320 320 501,978 493,523 320 474,112 +sse h1 - - - 0 556,518 - 0 386,061 +sse_small h1 - - - 0 607,330 - 0 578,910 +sse_big h1 - - - 0 395,686 - 0 154,630 +stream h2 99,004 17,441 - - 180,410 - - 239,324 +stream_small h2 424,130 1,078,120 - - 530,580 - - 647,860 +stream_big h2 280,890 51,104 - - 345,274 - - 405,274 +sse h2 - - - - 344,560 - - 515,411 +sse_small h2 - - - - 372,340 - - 562,190 +sse_big h2 - - - - 216,947 - - 138,618 + +not answering the case correctly + + mist h1 stream: 34 failed, 34 non-2xx + mist h1 stream_big: 1 failed, 1 non-2xx + chatterbox h2 stream_big: 22 failed, 0 non-2xx + +collapsed against peers + + ewe@4 h1 file_tiny: 5 req/s against a 39,110 median for this case + ewe@4 h1 file_small: 5 req/s against a 36,329 median for this case + ewe@4 h1 file_big: 5 req/s against a 3,395 median for this case + ewe@4 h1 stream: 5 req/s against a 33,801 median for this case + ewe@4 h1 stream_small: 5 req/s against a 6,540 median for this case + ewe@4 h1 stream_big: 5 req/s against a 6,639 median for this case + ewe@4 h1 sse: 0 req/s against a 12,010 median for this case + ewe@4 h1 sse_small: 0 req/s against a 5,746 median for this case + ewe@4 h1 sse_big: 0 req/s against a 2,403 median for this case + mist h1 stream: 6 req/s against a 33,801 median for this case + mist h1 stream_small: 5 req/s against a 6,540 median for this case + mist h1 stream_big: 5 req/s against a 6,639 median for this case + mist h1 sse: 0 req/s against a 12,010 median for this case + mist h1 sse_small: 0 req/s against a 5,746 median for this case + mist h1 sse_big: 0 req/s against a 2,403 median for this case + elli h1 stream: 5 req/s against a 33,801 median for this case + elli h1 stream_small: 5 req/s against a 6,540 median for this case + elli h1 stream_big: 5 req/s against a 6,639 median for this case + +repeats disagreed by over 5% + + ewe@5 h2 file_big: 7% apart across repeats + bandit h1 file_tiny: 6% apart across repeats + bandit h1 file_small: 6% apart across repeats + bandit h2 file_small: 7% apart across repeats + roadrunner h1 stream: 10% apart across repeats + roadrunner h2 file_big: 6% apart across repeats + roadrunner h2 stream_big: 5% apart across repeats + chatterbox h2 file_big: 64% apart across repeats + +results/20260812-151117-latency/latency.csv + +p99 at a fixed offered rate for http1 + +hello +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +50,000 2.46ms 1.87ms 2.19ms 1.78ms 3.43ms 2.13ms 1.92ms +100,000 3.67ms 2.84ms 3.20ms 2.74ms 8.57ms 3.53ms 3.34ms +150,000 29.02ms 4.14ms 5.02ms 3.91ms 3.03s 4.32ms 4.15ms + +hello_headers +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +50,000 2.85ms 2.02ms 2.53ms 1.99ms 3.25ms 2.40ms 2.12ms +100,000 4.16ms 3.83ms 3.90ms 3.39ms 1.95s 3.82ms 3.79ms +150,000 1.89s 4.25ms 476.16ms 4.04ms 4.74s 6.83ms 3.96ms + +echo_1kb +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +50,000 2.76ms 2.12ms 2.51ms 2.10ms 3.53ms 2.34ms 2.15ms +100,000 3.97ms 3.82ms 3.68ms 3.85ms 3.50s 3.95ms 3.88ms +150,000 1.55s 4.07ms 13.01ms 4.50ms 5.41s 6.44ms 4.34ms + +echo_1kb_headers +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +40,000 2.23ms 1.71ms 2.17ms 1.80ms 3.76ms 2.03ms 1.85ms +80,000 3.53ms 3.02ms 3.72ms 3.63ms 3.65s 3.47ms 3.39ms +120,000 818.69ms 4.30ms 7.92ms 3.98ms 5.82s 6.21ms 3.74ms + +echo_10kb +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +20,000 1.82ms 1.66ms 1.73ms 1.66ms 4.42s 1.71ms 1.67ms +60,000 3.59ms 3.13ms 3.23ms 2.64ms 8.01s 3.07ms 2.82ms +100,000 5.49ms 4.10ms 4.25ms 3.83ms 8.67s 4.12ms 3.88ms + +echo_chunked_10kb +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +40,000 3.45ms - 3.07ms 2.20ms - 2.66ms 2.22ms +70,000 7.57ms - 3.94ms 3.63ms - 3.57ms 4.03ms +100,000 2.82s - 297.21ms 4.44ms - 7.37ms 4.10ms + +file_tiny +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +20,000 3.68ms 2.95ms - 3.29ms 4.15ms 3.93ms 2.83ms +40,000 13.85ms 182.53ms - 4.74ms 1.27s 664.58ms 567.81ms +60,000 2.35s 3.32s - 74.50ms 4.15s 3.61s 3.55s + +file_small +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +15,000 3.38ms 2.15ms - 2.35ms 2.69ms 2.37ms 2.25ms +30,000 4.82ms 4.98ms - 4.52ms 79.42ms 4.89ms 5.37ms +45,000 439.55ms 1.90s - 8.90ms 3.20s 2.13s 2.13s + +file_big +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +500 4.53ms 4.43ms - 4.41ms 6.27ms 17.93ms 4.66ms +1,000 4.64ms 4.58ms - 4.33ms 7.04ms 4.54ms 208.26ms +1,500 4.91ms 4.71ms - 4.82ms 960.00ms 4.93ms 4.83ms + +stream +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +20,000 1.99ms - - 1.75ms 1.94ms - 2.40ms +40,000 3.21ms - - 2.45ms 3.08ms - 1.81s +60,000 4.17ms - - 3.47ms 3.72ms - 4.18s + +stream_small +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +3,000 3.79ms - - 3.48ms 3.34ms - 3.74ms +5,000 10.86ms - - 5.09ms 4.90ms - 11.33ms +7,000 1.91s - - 25.12ms 133.50ms - 1.25s + +stream_big +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +3,000 3.61ms - - 3.33ms 3.15ms - 3.48ms +5,000 8.70ms - - 5.17ms 4.63ms - 5.67ms +7,000 1.39s - - 87.81ms 21.45ms - 92.80ms + +sse +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +6,000 - - 3.39ms 2.64ms - 3.20ms 2.87ms +10,000 - - 906.75ms 4.26ms - 612.35ms 8.88ms +14,000 - - 3.34s 9.06ms - 2.94s 1.71s + +sse_small +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +3,000 - - 5.03ms 3.98ms - 5.55ms 3.92ms +5,000 - - 1.47s 14.72ms - 1.32s 16.91ms +7,000 - - 3.63s 2.32s - 3.87s 1.84s + +sse_big +target req/s bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------- +3,000 - - 3.99ms 3.58ms - 4.03ms 2.13s +5,000 - - 236.41ms 13.49ms - 52.38ms 5.17s +7,000 - - 2.78s 2.49s - 2.73s 6.52s + +CO gap at the highest rate + +case bandit elli ewe@4 ewe@5 httpd mist roadrunner +-------------------------------------------------------------------------------------------------------- +hello 49.5x 8.8x 7.0x 9.7x 3792.2x 6.7x 7.9x +hello_headers 3181.8x 6.8x 813.9x 7.1x 4989.5x 11.1x 6.0x +echo_1kb 2500.0x 6.7x 19.9x 6.3x 5252.4x 10.4x 5.9x +echo_1kb_headers 1106.3x 6.3x 10.4x 5.5x 4409.1x 6.7x 4.9x +echo_10kb 5.5x 5.5x 4.1x 6.3x 1723.7x 5.1x 6.3x +echo_chunked_10kb 2563.6x - 333.9x 4.5x - 5.8x 4.5x +file_tiny 525.7x 1057.3x - 17.1x 1596.2x 1262.2x 1163.9x +file_small 84.4x 627.1x - 2.3x 1077.4x 727.0x 659.4x +file_big 1.3x 1.3x - 1.3x 19.1x 1.3x 1.3x +stream 2.8x - - 4.8x 3.6x - 2100.5x +stream_small 125.2x - - 2.3x 11.9x - 57.2x +stream_big 95.4x - - 7.4x 2.2x - 6.1x +sse - - 510.7x 1.9x - 455.8x 176.7x +sse_small - - 192.5x 153.8x - 195.6x 75.9x +sse_big - - 197.7x 164.5x - 195.8x 119.5x \ No newline at end of file diff --git a/benchmark/roadrunner/src/roadrunner_bench_app.erl b/benchmark/roadrunner/src/roadrunner_bench_app.erl index 0965952..b102d99 100644 --- a/benchmark/roadrunner/src/roadrunner_bench_app.erl +++ b/benchmark/roadrunner/src/roadrunner_bench_app.erl @@ -2,10 +2,24 @@ -behaviour(application). --export([start/2, stop/1]). +-export([start/2, stop/1, payload/1]). + +-define(SMALL_LINE, ~"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"). +-define(BIG_REPEATS, 256). start(_StartType, _StartArgs) -> + BigLine = binary:copy(?SMALL_LINE, ?BIG_REPEATS), + put_payload(small_chunk, ?SMALL_LINE), + put_payload(big_chunk, BigLine), + put_payload(small_line, ?SMALL_LINE), + put_payload(big_line, BigLine), roadrunner_bench_sup:start_link(). stop(_State) -> ok. + +put_payload(Key, Value) -> + persistent_term:put({?MODULE, Key}, Value). + +payload(Key) -> + persistent_term:get({?MODULE, Key}). diff --git a/benchmark/roadrunner/src/roadrunner_bench_callback.erl b/benchmark/roadrunner/src/roadrunner_bench_callback.erl index 86c9c98..7331dc0 100644 --- a/benchmark/roadrunner/src/roadrunner_bench_callback.erl +++ b/benchmark/roadrunner/src/roadrunner_bench_callback.erl @@ -7,6 +7,8 @@ -include_lib("kernel/include/file.hrl"). -define(SSE_EVENTS, 32). +-define(SMALL_COUNT, 100). +-define(BIG_COUNT, 64). handle(Req) -> route(roadrunner_req:method(Req), roadrunner_req:path(Req), Req). @@ -29,40 +31,48 @@ route(~"GET", ~"/stream", Req) -> end}, {Resp, Req}; -route(~"GET", ~"/sse", Req) -> - Headers = [ - {~"content-type", ~"text/event-stream"}, - {~"cache-control", ~"no-cache"} - ], - Resp = {stream, 200, Headers, fun(Send) -> start_sse(Send) end}, - {Resp, Req}; +route(~"GET", ~"/stream/small", Req) -> + {burst(roadrunner_bench_app:payload(small_chunk), ?SMALL_COUNT), Req}; +route(~"GET", ~"/stream/big", Req) -> + {burst(roadrunner_bench_app:payload(big_chunk), ?BIG_COUNT), Req}; +route(~"GET", ~"/sse", Req) -> + {sse_burst(roadrunner_bench_app:payload(small_line), ?SSE_EVENTS), Req}; +route(~"GET", ~"/sse/small", Req) -> + {sse_burst(roadrunner_bench_app:payload(small_line), ?SMALL_COUNT), Req}; +route(~"GET", ~"/sse/big", Req) -> + {sse_burst(roadrunner_bench_app:payload(big_line), ?BIG_COUNT), Req}; + +route(~"GET", ~"/file/tiny", Req) -> + send_file(Req, "../priv/file_1kb.bin"); route(~"GET", ~"/file/small", Req) -> send_file(Req, "../priv/file_100kb.bin"); route(~"GET", ~"/file/big", Req) -> - send_file(Req, "../priv/file_1gb.bin"); + send_file(Req, "../priv/file_5mb.bin"); route(_Method, _Path, Req) -> {roadrunner_resp:not_found(), Req}. -start_sse(Send) -> - self() ! {sse_tick, 1}, - send_sse(Send). - -send_sse(Send) -> - receive - {sse_tick, N} when N >= ?SSE_EVENTS -> - Send(sse_event(N), fin); - {sse_tick, N} -> - Send(sse_event(N), nofin), - self() ! {sse_tick, N + 1}, - send_sse(Send) - end. +sse_headers() -> + [{~"content-type", ~"text/event-stream"}, {~"cache-control", ~"no-cache"}]. + +burst(Chunk, Count) -> + {stream, 200, [], fun(Send) -> send_burst(Send, Chunk, Count) end}. + +send_burst(Send, Chunk, 1) -> + Send(Chunk, fin); +send_burst(Send, Chunk, Remaining) -> + Send(Chunk, nofin), + send_burst(Send, Chunk, Remaining - 1). + +sse_burst(Data, Count) -> + {stream, 200, sse_headers(), fun(Send) -> send_events(Send, Data, Count) end}. -sse_event(N) -> - Id = integer_to_binary(N), - Data = <<"{\"n\":", Id/binary, ",\"at\":\"benchmark\"}">>, - iolist_to_binary(roadrunner_sse:event(~"tick", Data, Id)). +send_events(Send, Data, 1) -> + Send(roadrunner_sse:event(~"tick", Data), fin); +send_events(Send, Data, Remaining) -> + Send(roadrunner_sse:event(~"tick", Data), nofin), + send_events(Send, Data, Remaining - 1). read_all_chunks(Req, Acc) -> case roadrunner_req:read_body_chunked(Req) of diff --git a/benchmark/roadrunner/src/roadrunner_bench_sup.erl b/benchmark/roadrunner/src/roadrunner_bench_sup.erl index 6120a9b..3e386c3 100644 --- a/benchmark/roadrunner/src/roadrunner_bench_sup.erl +++ b/benchmark/roadrunner/src/roadrunner_bench_sup.erl @@ -11,21 +11,31 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). init([]) -> - {ok, _Pid} = roadrunner:start_listener(bench_listener, #{ - port => 3007, + {ok, _Http1} = roadrunner:start_listener(bench_http1, listener_opts(3007, #{})), + {ok, _Http2} = roadrunner:start_listener(bench_http2, + listener_opts(3008, #{protocols => [http2]})), + + SupFlags = #{strategy => one_for_one, + intensity => 0, + period => 1}, + {ok, {SupFlags, []}}. + +listener_opts(Port, Extra) -> + maps:merge(#{ + port => Port, body_buffering => manual, routes => [ {~"/hello", roadrunner_bench_callback, undefined}, {~"/echo", roadrunner_bench_callback, undefined}, {~"/echo/chunked", roadrunner_bench_callback, undefined}, {~"/stream", roadrunner_bench_callback, undefined}, + {~"/stream/small", roadrunner_bench_callback, undefined}, + {~"/stream/big", roadrunner_bench_callback, undefined}, {~"/sse", roadrunner_bench_callback, undefined}, + {~"/sse/small", roadrunner_bench_callback, undefined}, + {~"/sse/big", roadrunner_bench_callback, undefined}, + {~"/file/tiny", roadrunner_bench_callback, undefined}, {~"/file/small", roadrunner_bench_callback, undefined}, {~"/file/big", roadrunner_bench_callback, undefined} ] - }), - - SupFlags = #{strategy => one_for_one, - intensity => 0, - period => 1}, - {ok, {SupFlags, []}}. + }, Extra). diff --git a/benchmark/throughput.sh b/benchmark/throughput.sh new file mode 100755 index 0000000..85ab617 --- /dev/null +++ b/benchmark/throughput.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# Usage: ./throughput.sh [--servers ewe@5,bandit] [--cases sse_big] +# [--profiles h1,h2,h2-serial] + +set -u + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT" +source "$ROOT/config.sh" +source "$ROOT/lib.sh" + +ONLY_SERVERS="" +ONLY_CASES="" +ONLY_PROFILES="$DEFAULT_PROFILES" +while [ $# -gt 0 ]; do + case "$1" in + --servers) ONLY_SERVERS="$2"; shift 2 ;; + --cases) ONLY_CASES="$2"; shift 2 ;; + --profiles) ONLY_PROFILES="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +command -v h2load > /dev/null || { echo "h2load not found (install nghttp2)" >&2; exit 1; } + +ensure_fixtures +new_results_dir throughput +CSV="$RESULTS_DIR/throughput.csv" +echo "server,profile,protocol,connections,streams,case,repeat,requests_per_sec,messages,messages_per_sec,mb_per_sec,succeeded,failed,non_2xx" > "$CSV" + +{ + echo "date $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "load h2load, ${THREADS} threads" + for profile_entry in "${PROFILES[@]}"; do + IFS='|' read -r profile protocol connections streams <<< "$profile_entry" + selected "$profile" "$ONLY_PROFILES" || continue + printf ' %-10s %s, %s connections x %s stream(s)\n' \ + "$profile" "$protocol" "$connections" "$streams" + done + echo "duration ${DURATION}s measured, ${WARMUP}s warmup, ${REPEATS} repeats per cell" + echo "cpu server on ${SERVER_CPUS}, load generator on ${LOAD_CPUS}" +} | tee "$RESULTS_DIR/run.txt" +echo + +set_protocol_args() { + case "$1" in + h1) PROTOCOL_ARGS=(--h1) ;; + h2) PROTOCOL_ARGS=(-p h2c) ;; + *) echo "unknown protocol: $1" >&2; exit 1 ;; + esac +} + +parse_h2load() { + awk ' + /^finished in/ { gsub(",", ""); rate = $4; mb = $6; sub("MB/s", "", mb); finished = 1 } + /^requests:/ { gsub(",", ""); succeeded = $8; failed = $10 + $12 + $14 } + /^status codes:/ { gsub(",", ""); non2xx = $5 + $7 + $9 } + END { + if (!finished) exit 1 + printf "%s|%s|%s|%s|%s\n", rate + 0, mb + 0, succeeded + 0, failed + 0, non2xx + 0 + } + ' +} + +run_case() { + local repeat="$1" body_path raw parsed + local rate mb succeeded failed non2xx messages_per_sec + + set_protocol_args "$protocol" + local args=("${PROTOCOL_ARGS[@]}" -c "$connections" -m "$streams" -t "$THREADS" -D "$DURATION") + [ "$headers" = "yes" ] && args+=("${HEADER_ARGS[@]}") + + body_path="$(body_file "$body")" + [ -n "$body_path" ] && args+=(-d "$body_path") + + raw="$RESULTS_DIR/${server}__${profile}__${case_name}__${repeat}.txt" + if ! parsed="$("${PIN_LOAD[@]}" h2load "${args[@]}" "http://127.0.0.1:$port$path" 2>&1 \ + | tee "$raw" | parse_h2load)"; then + printf 'failed ' + return 1 + fi + + IFS='|' read -r rate mb succeeded failed non2xx <<< "$parsed" + messages_per_sec="$(awk -v r="$rate" -v m="$messages" 'BEGIN { printf "%.0f", r * m }')" + + echo "$server,$profile,$protocol,$connections,$streams,$case_name,$repeat,$rate,$messages,$messages_per_sec,$mb,$succeeded,$failed,$non2xx" >> "$CSV" + printf '%s ' "$rate" +} + +for server_entry in "${SERVERS[@]}"; do + IFS='|' read -r server dir cmd h1_port h2_port <<< "$server_entry" + selected "$server" "$ONLY_SERVERS" || continue + + echo "=== $server ===" + start_server "$dir" "$cmd" "$(startup_port "$h1_port" "$h2_port")" "$RESULTS_DIR/$server.log" || continue + + for profile_entry in "${PROFILES[@]}"; do + IFS='|' read -r profile protocol connections streams <<< "$profile_entry" + selected "$profile" "$ONLY_PROFILES" || continue + + if [ "$protocol" = h1 ]; then + port="$h1_port" + else + port="$h2_port" + fi + [ "$port" = "-" ] && continue + + set_protocol_args "$protocol" + "${PIN_LOAD[@]}" h2load "${PROTOCOL_ARGS[@]}" \ + -c "$connections" -m "$streams" -t "$THREADS" -D "$WARMUP" \ + "http://127.0.0.1:$port/hello" > /dev/null 2>&1 + + for case_entry in "${CASES[@]}"; do + IFS='|' read -r case_name path body headers messages <<< "$case_entry" + selected "$case_name" "$ONLY_CASES" || continue + + if unsupported "$server" "$case_name"; then + printf ' %-10s %-20s not supported\n' "$profile" "$case_name" + continue + fi + + printf ' %-10s %-20s ' "$profile" "$case_name" + for ((repeat = 1; repeat <= REPEATS; repeat++)); do + run_case "$repeat" + done + echo "req/s" + done + done + + stop_server + echo +done + +render_report "$CSV" diff --git a/benchmark/wrk/get_headers.lua b/benchmark/wrk/get_headers.lua new file mode 100644 index 0000000..28cb047 --- /dev/null +++ b/benchmark/wrk/get_headers.lua @@ -0,0 +1,5 @@ +wrk.headers["cookie"] = "session=abc123def456; theme=dark; locale=en-US" +wrk.headers["x-request-id"] = "9f86d081-b1bb-4c2e-8f21-9c7b1f2a0e3e" +wrk.headers["x-client-version"] = "4.2.1" +wrk.headers["accept-language"] = "en-US,en;q=0.9" +wrk.headers["x-forwarded-for"] = "203.0.113.42" diff --git a/benchmark/wrk/post_echo.lua b/benchmark/wrk/post_echo.lua deleted file mode 100644 index bc52e7b..0000000 --- a/benchmark/wrk/post_echo.lua +++ /dev/null @@ -1,6 +0,0 @@ --- POST /echo with a fixed Content-Length body, exercising the buffered --- (whole-body) request read path. - -wrk.method = "POST" -wrk.body = string.rep("a", 4096) -wrk.headers["Content-Type"] = "application/octet-stream" diff --git a/benchmark/wrk/post_echo_10kb.lua b/benchmark/wrk/post_echo_10kb.lua new file mode 100644 index 0000000..2aa44e4 --- /dev/null +++ b/benchmark/wrk/post_echo_10kb.lua @@ -0,0 +1,3 @@ +wrk.method = "POST" +wrk.body = string.rep("a", 10240) +wrk.headers["Content-Type"] = "application/octet-stream" diff --git a/benchmark/wrk/post_echo_1kb.lua b/benchmark/wrk/post_echo_1kb.lua new file mode 100644 index 0000000..682add0 --- /dev/null +++ b/benchmark/wrk/post_echo_1kb.lua @@ -0,0 +1,3 @@ +wrk.method = "POST" +wrk.body = string.rep("a", 1024) +wrk.headers["Content-Type"] = "application/octet-stream" diff --git a/benchmark/wrk/post_echo_1kb_headers.lua b/benchmark/wrk/post_echo_1kb_headers.lua new file mode 100644 index 0000000..0a111b3 --- /dev/null +++ b/benchmark/wrk/post_echo_1kb_headers.lua @@ -0,0 +1,8 @@ +wrk.method = "POST" +wrk.body = string.rep("a", 1024) +wrk.headers["Content-Type"] = "application/octet-stream" +wrk.headers["cookie"] = "session=abc123def456; theme=dark; locale=en-US" +wrk.headers["x-request-id"] = "9f86d081-b1bb-4c2e-8f21-9c7b1f2a0e3e" +wrk.headers["x-client-version"] = "4.2.1" +wrk.headers["accept-language"] = "en-US,en;q=0.9" +wrk.headers["x-forwarded-for"] = "203.0.113.42" diff --git a/benchmark/wrk/post_echo_chunked.lua b/benchmark/wrk/post_echo_chunked_10kb.lua similarity index 53% rename from benchmark/wrk/post_echo_chunked.lua rename to benchmark/wrk/post_echo_chunked_10kb.lua index abbe047..4d05c6a 100644 --- a/benchmark/wrk/post_echo_chunked.lua +++ b/benchmark/wrk/post_echo_chunked_10kb.lua @@ -1,18 +1,8 @@ --- POST /echo/chunked with a Transfer-Encoding: chunked body, exercising the --- incremental/streamed request read path. wrk has no built-in support for --- chunked request bodies, so the raw request is hand-built here. - wrk.method = "POST" -local chunks = { - string.rep("a", 1024), - string.rep("b", 1024), - string.rep("c", 1024), - string.rep("d", 1024), -} - local encoded = "" -for _, chunk in ipairs(chunks) do +for _ = 1, 10 do + local chunk = string.rep("a", 1024) encoded = encoded .. string.format("%x", #chunk) .. "\r\n" .. chunk .. "\r\n" end encoded = encoded .. "0\r\n\r\n" diff --git a/benchmark/wrk2-setup.sh b/benchmark/wrk2-setup.sh index fa623c1..ed8ede3 100755 --- a/benchmark/wrk2-setup.sh +++ b/benchmark/wrk2-setup.sh @@ -1,12 +1,4 @@ #!/usr/bin/env bash -# -# Clones and builds wrk2 (https://github.com/giltene/wrk2) into benchmark/.wrk2/, -# for use by bench.sh. -# -# On newer binutils 2.4x+, wrk2's vendored LuaJIT bytecode-to-object step links -# fine on its own, but the final link fails with "string table is corrupt" using -# the default bfd linker. I don't know if it's issue on my side, but building -# with the gold linker avoids it. set -eu diff --git a/src/ewe/internal/sse_ffi.erl b/src/ewe/internal/sse_ffi.erl index 752b720..138b37a 100644 --- a/src/ewe/internal/sse_ffi.erl +++ b/src/ewe/internal/sse_ffi.erl @@ -15,7 +15,20 @@ init() -> ok. split_breaks(Bin) -> - binary:split(Bin, persistent_term:get({?MODULE, break}), [global]). + case has_break(Bin) of + false -> [Bin]; + true -> binary:split(Bin, break_pattern(), [global]) + end. strip_breaks(Bin) -> - binary:replace(Bin, persistent_term:get({?MODULE, break}), <<>>, [global]). + case has_break(Bin) of + false -> Bin; + true -> binary:replace(Bin, break_pattern(), <<>>, [global]) + end. + +break_pattern() -> + persistent_term:get({?MODULE, break}). + +has_break(Bin) -> + binary:match(Bin, <<"\n">>) =/= nomatch orelse + binary:match(Bin, <<"\r">>) =/= nomatch. -- 2.51.2