From 9b79f7b8de4d34e31e47ecc380988b5c17f4f7eb Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Mon, 3 Aug 2026 02:44:21 +0300 Subject: [PATCH] Measure PostgreSQL rate limiter --- README.md | 12 ++ docs/performance.md | 23 ++++ lib/mix/tasks/wnh.rate_limit_benchmark.ex | 161 ++++++++++++++++++++++ scripts/rate-limit-benchmark.sh | 142 +++++++++++++++++++ 4 files changed, 338 insertions(+) create mode 100644 lib/mix/tasks/wnh.rate_limit_benchmark.ex create mode 100755 scripts/rate-limit-benchmark.sh diff --git a/README.md b/README.md index 444e001..6ac0e11 100644 --- a/README.md +++ b/README.md @@ -620,6 +620,18 @@ default, while an explicit `{}` disables all counters for isolated load/E2E runs. These values are an initial product policy, not universal security or capacity thresholds. Supported actions are listed in `docs/trust-safety.md`. +Measure the limiter on a disposable PostgreSQL database and a one-CPU +application container by supplying the workload explicitly: + +```bash +./scripts/rate-limit-benchmark.sh ATTEMPTS CONCURRENCY DISTRIBUTED_SCOPES +``` + +The script measures a single contended bucket and a distributed-scope profile, +writes JSON under ignored `output/rate-limit/`, and removes only its uniquely +named Compose project, volume, containers, and images on success, failure, or +interrupt. It does not infer a production limit from the result. + ## Local Kubernetes verification The kind script downloads checksum-verified kubectl, Helm, and kind binaries diff --git a/docs/performance.md b/docs/performance.md index 98b9034..332d628 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -438,6 +438,29 @@ Ignored evidence: - `output/performance/production-http-db-stats-20260728/` - `output/performance/production-ws-heartbeat-final-20260728/` +## Isolated rate-limiter observation + +Observed locally on 2026-08-03 with the checked-in +`scripts/rate-limit-benchmark.sh`. The application container was limited to one +CPU; PostgreSQL reported version 18.4. Both profiles used 5,000 calls with +concurrency 20. The first profile deliberately contended on one bucket; the +second used 5,000 independent scopes. + +| Profile | Successful / failed / limited | Elapsed | Observed rate | Buckets | +| --- | ---: | ---: | ---: | ---: | +| One hot scope | 5,000 / 0 / 0 | 12,769.306 ms | 391.56 operations/s | 1 | +| Distributed scopes | 5,000 / 0 / 0 | 1,312.608 ms | 3,809.21 operations/s | 5,000 | + +Each run persisted exactly 5,000 increments. The task then deleted its unique +benchmark action: cleanup left zero benchmark buckets. The generated JSON is +kept in the ignored local directory +`output/rate-limit/20260802233523-2242041/`. + +This observation measures the atomic counter implementation on this local +host. It is not a production capacity claim and does not justify a hard startup +guard. The much slower hot-scope result is expected contention on one database +row; ordinary email/IP policy traffic distributes attempts across many scopes. + ## Observed local comparison Observed on 2026-07-18 with Docker Engine 29.6.2 on the recorded 32-CPU, diff --git a/lib/mix/tasks/wnh.rate_limit_benchmark.ex b/lib/mix/tasks/wnh.rate_limit_benchmark.ex new file mode 100644 index 0000000..241c7fa --- /dev/null +++ b/lib/mix/tasks/wnh.rate_limit_benchmark.ex @@ -0,0 +1,161 @@ +defmodule Mix.Tasks.Wnh.RateLimitBenchmark do + use Mix.Task + + import Ecto.Query + + alias WhoNeedHelp.Repo + alias WhoNeedHelp.Trust.{RateLimitBucket, RateLimiter} + + @shortdoc "Measures the PostgreSQL rate limiter in an isolated database" + + @moduledoc """ + Measures the shared PostgreSQL limiter without choosing or enforcing a + product threshold. + + The caller must provide the workload explicitly. Every run uses a unique + action name, restores the previous application policy, and deletes only the + buckets created by that action before it exits. + + mix wnh.rate_limit_benchmark \ + --profile hot \ + --attempts 5000 \ + --concurrency 20 \ + --scopes 1 \ + --output /benchmark-output/hot.json + """ + + @impl Mix.Task + def run(arguments) do + Mix.Task.run("app.start") + + {options, rest, invalid} = + OptionParser.parse(arguments, + strict: [ + profile: :string, + attempts: :integer, + concurrency: :integer, + scopes: :integer, + output: :string + ] + ) + + if rest != [] or invalid != [], + do: Mix.raise("invalid arguments: #{inspect(rest ++ invalid)}") + + profile = Keyword.get(options, :profile) || Mix.raise("--profile is required") + attempts = positive!(options, :attempts) + concurrency = positive!(options, :concurrency) + scopes = positive!(options, :scopes) + output = Keyword.get(options, :output) || Mix.raise("--output PATH is required") + + unless profile in ["hot", "distributed"], + do: Mix.raise("--profile must be hot or distributed") + + if profile == "hot" and scopes != 1, + do: Mix.raise("the hot profile requires --scopes 1") + + if scopes > attempts, do: Mix.raise("--scopes cannot exceed --attempts") + + if Mix.env() == :prod, + do: Mix.raise("this benchmark refuses to run with MIX_ENV=prod") + + if Mix.env() == :test do + Ecto.Adapters.SQL.Sandbox.mode(Repo, :auto) + end + + action = "rate_limit_benchmark_" <> String.replace(Ecto.UUID.generate(), "-", "") + previous_policies = Application.get_env(:who_need_help, :rate_limit_policies, %{}) + + File.mkdir_p!(Path.dirname(output)) + + try do + Application.put_env(:who_need_help, :rate_limit_policies, %{ + action => %{limit: attempts + 1, window_seconds: 3_600} + }) + + {elapsed_native, results} = + :timer.tc(fn -> + 1..attempts + |> Task.async_stream( + fn index -> + scope = "benchmark-scope-#{rem(index - 1, scopes)}" + RateLimiter.check(action, scope) + end, + max_concurrency: concurrency, + ordered: false, + timeout: :infinity + ) + |> Enum.to_list() + end) + + counts = summarize_results(results) + + bucket_count = + Repo.aggregate(from(bucket in RateLimitBucket, where: bucket.action == ^action), :count) + + total_count = + Repo.one!( + from bucket in RateLimitBucket, + where: bucket.action == ^action, + select: coalesce(sum(bucket.count), 0) + ) + + elapsed_seconds = elapsed_native / 1_000_000 + + summary = %{ + profile: profile, + attempts: attempts, + concurrency: concurrency, + scopes: scopes, + elapsed_ms: Float.round(elapsed_seconds * 1_000, 3), + operations_per_second: Float.round(attempts / elapsed_seconds, 2), + successful_calls: counts.ok, + rate_limited_calls: counts.rate_limited, + failed_tasks: counts.failed, + buckets_created: bucket_count, + persisted_attempt_count: total_count, + scheduler_count: System.schedulers_online(), + postgres_version: Repo.query!("SHOW server_version").rows |> hd() |> hd(), + captured_at: DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() + } + + {deleted, _} = delete_benchmark_rows(action) + + remaining = + Repo.aggregate(from(bucket in RateLimitBucket, where: bucket.action == ^action), :count) + + final_summary = + Map.merge(summary, %{cleanup_deleted: deleted, cleanup_remaining: remaining}) + + File.write!(output, Jason.encode_to_iodata!(final_summary, pretty: true)) + Mix.shell().info(Jason.encode!(final_summary)) + + if counts.rate_limited != 0 or counts.failed != 0 or total_count != attempts or + remaining != 0 do + Mix.raise("rate limiter benchmark correctness checks failed") + end + after + delete_benchmark_rows(action) + Application.put_env(:who_need_help, :rate_limit_policies, previous_policies) + end + end + + defp positive!(options, key) do + case Keyword.get(options, key) do + value when is_integer(value) and value > 0 -> value + _ -> Mix.raise("--#{key} must be a positive integer") + end + end + + defp summarize_results(results) do + Enum.reduce(results, %{ok: 0, rate_limited: 0, failed: 0}, fn + {:ok, {:ok, _}}, counts -> Map.update!(counts, :ok, &(&1 + 1)) + {:ok, {:error, :rate_limited}}, counts -> Map.update!(counts, :rate_limited, &(&1 + 1)) + _other, counts -> Map.update!(counts, :failed, &(&1 + 1)) + end) + end + + defp delete_benchmark_rows(action) do + Repo.delete_all(from bucket in RateLimitBucket, where: bucket.action == ^action) + end +end diff --git a/scripts/rate-limit-benchmark.sh b/scripts/rate-limit-benchmark.sh new file mode 100755 index 0000000..0f0af53 --- /dev/null +++ b/scripts/rate-limit-benchmark.sh @@ -0,0 +1,142 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +if [ "$#" -ne 3 ]; then + echo "usage: $0 ATTEMPTS CONCURRENCY DISTRIBUTED_SCOPES" >&2 + exit 2 +fi + +attempts=$1 +concurrency=$2 +distributed_scopes=$3 + +for value in "$attempts" "$concurrency" "$distributed_scopes"; do + case "$value" in + ''|*[!0-9]*) + echo "all workload arguments must be positive integers" >&2 + exit 2 + ;; + esac + + if [ "$value" -lt 1 ]; then + echo "all workload arguments must be positive integers" >&2 + exit 2 + fi +done + +if [ "$distributed_scopes" -gt "$attempts" ]; then + echo "DISTRIBUTED_SCOPES cannot exceed ATTEMPTS" >&2 + exit 2 +fi + +run_id="$(date -u +%Y%m%d%H%M%S)-$$" +project="wnh_rate_limit_$(printf '%s' "$run_id" | tr -d '-')" +image="who-need-help:rate-limit-$run_id" +postgis_image="who-need-help:rate-limit-postgis-$run_id" +output="$ROOT/output/rate-limit/$run_id" +compose_env="$(mktemp "${TMPDIR:-/tmp}/wnh-rate-limit-compose.XXXXXX")" +runtime_env="$(mktemp "${TMPDIR:-/tmp}/wnh-rate-limit-runtime.XXXXXX")" + +umask 077 +db_user="wnh_rate_$(openssl rand -hex 6)" +db_password=$(openssl rand -base64 48 | tr -d '\n') + +cat >"$compose_env" <"$runtime_env" </dev/null 2>&1 || cleanup_status=$? + rm -f "$compose_env" "$runtime_env" || cleanup_status=$? + + for candidate in "$image" "$postgis_image"; do + if docker image inspect "$candidate" >/dev/null 2>&1; then + if image_tag_is_referenced "$candidate"; then + echo "Refusing to remove referenced benchmark image: $candidate" >&2 + cleanup_status=1 + else + docker image rm "$candidate" >/dev/null || cleanup_status=$? + fi + fi + done + + if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then + status=$cleanup_status + fi + + exit "$status" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$output" + +docker build --target test --tag "$image" . +docker build --tag "$postgis_image" --file Dockerfile.postgis . +$compose up --detach --wait db + +run_mix() { + docker run --rm \ + --cpus 1 \ + --network "${project}_internal" \ + --env-file "$runtime_env" \ + --volume "$output:/benchmark-output" \ + "$image" "$@" +} + +run_mix mix ecto.create --quiet +run_mix mix ecto.migrate --quiet + +run_mix mix wnh.rate_limit_benchmark \ + --profile hot \ + --attempts "$attempts" \ + --concurrency "$concurrency" \ + --scopes 1 \ + --output /benchmark-output/hot.json + +run_mix mix wnh.rate_limit_benchmark \ + --profile distributed \ + --attempts "$attempts" \ + --concurrency "$concurrency" \ + --scopes "$distributed_scopes" \ + --output /benchmark-output/distributed.json + +docker run --rm \ + --volume "$output:/output" \ + alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 \ + chown -R "$(id -u):$(id -g)" /output + +echo "Rate limiter measurements: $output"