Keep million-row fixtures in development

2026-07-28 19:08:34 +03:00 · 2026-07-28 19:08:34 +03:00 · 4490280e0e
commit 4490280e0e
parent ebaaeaf517
5 changed files with 299 additions and 9 deletions
--- a/.env.example
+++ b/.env.example
@ -183,6 +183,10 @@ FCM_SERVICE_ACCOUNT_JSON_BASE64=
 POSTGRES_DB=who_need_help
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=replace-with-a-local-or-deployment-secret
+
+# Required only by scripts/dev-scale-seed.sh for the local synthetic viewer.
+# Generate a distinct value; never reuse a real user or deployment password.
+DEV_SCALE_VIEWER_PASSWORD=replace-with-a-random-local-fixture-password
 DATABASE_URL=ecto://postgres:replace-with-url-encoded-password@db/who_need_help
 # Optional absolute host directory containing PostgreSQL Unix sockets. When it
 # is set in external mode, Compose mounts it read-only and Ecto uses it instead
--- a/docs/performance.md
+++ b/docs/performance.md
@ -767,7 +767,9 @@ ignored `output/runtime/scale.env`. Fixture mutation additionally requires the
 exact database name and `WNH_SCALE_FIXTURE_CONFIRM=persistent-local-scale`.
 The seed task refuses a partially populated database instead of mixing data
 sets. `scale-stack-stop.sh` stops only this Compose project and deliberately
-preserves the named volume.
+preserves the named volume. `scale-stack-down.sh` removes only the profile's
+containers and networks while also preserving that volume for an explicit
+operator decision.

 The canonical local fixture contains 1,000,000 requests, 1,000,000 activities,
 1,000 synthetic request owners, and one scale viewer. Records are distributed
@ -848,6 +850,26 @@ messages, and rate-limit buckets before and after.

 The final 2026-07-28 one-VU, 30-second, one-CPU static-shell observation
 completed 60 authenticated GETs with 0 HTTP failures and 120/120 checks.
+
+### Keeping the fixtures in the ordinary development site
+
+The isolated profile is useful for destructive load rehearsals. For routine
+development of the actual `whoneedhelp.imalto.site` UI, the same deterministic
+fixtures can instead be appended to the already running `who_need_help`
+development database:
+
+```sh
+./scripts/dev-scale-seed.sh
+```
+
+This guarded path requires the running Compose project and services to be named
+`who_need_help`, `who_need_help-web-1`, and `who_need_help-db-1`; verifies that
+the target database is exactly `who_need_help`; and uses the separate
+confirmation value `persistent-dev-scale`. It preserves ordinary development
+users and records. Synthetic rows carry `structured_data.scale_fixture=true`,
+use deterministic UUIDs, and are inserted with conflict handling, so rerunning
+the command fills missing fixtures without duplicating them. The resulting
+summary is ignored at `output/dev-scale/summary.json`.
 Overall HTTP duration was 2.566 ms median, 3.196 ms average, and 8.709 ms
 maximum. Every tracked table count was identical before and after. The
 discovery-query medians are the independently measured values in the table
--- a/lib/mix/tasks/wnh.scale_fixtures.ex
+++ b/lib/mix/tasks/wnh.scale_fixtures.ex
@ -5,8 +5,9 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
  alias WhoNeedHelp.{Activities, Catalog, Help, Repo}
  alias WhoNeedHelp.Help.DiscoveryViewport

-  @shortdoc "Prepares or verifies a persistent, isolated local discovery-scale data set"
-  @confirmation "persistent-local-scale"
+  @shortdoc "Prepares, appends, or verifies a persistent local discovery-scale data set"
+  @isolated_confirmation "persistent-local-scale"
+  @additive_confirmation "persistent-dev-scale"
  @default_rows 1_000_000
  @synthetic_users 1_000
  @viewer_email "scale-viewer@example.invalid"
@ -24,7 +25,7 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do

    action =
      case rest do
-        [value] when value in ["prepare", "verify"] -> value
+        [value] when value in ["prepare", "append", "verify"] -> value
        _other -> Mix.raise(usage())
      end

@ -32,16 +33,17 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
    activity_count = positive_count!(options, :activities)
    samples = positive_count!(options, :samples, 3)
    output = Keyword.get(options, :output) || Mix.raise("--output PATH is required")
-    context = verified_context!(output)
+    context = verified_context!(output, action)

    case action do
      "prepare" -> prepare!(context, request_count, activity_count)
+      "append" -> append!(context, request_count, activity_count)
      "verify" -> verify!(context, request_count, activity_count, samples)
    end
  end

  defp usage do
-    "usage: mix wnh.scale_fixtures prepare|verify " <>
+    "usage: mix wnh.scale_fixtures prepare|append|verify " <>
      "[--requests COUNT] [--activities COUNT] [--samples COUNT] --output /output/FILE.json"
  end

@ -50,12 +52,15 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
    if is_integer(value) and value > 0, do: value, else: Mix.raise("--#{key} must be positive")
  end

-  defp verified_context!(output) do
+  defp verified_context!(output, action) do
    expected_database = required_env!("WNH_SCALE_EXPECTED_DATABASE")
    output = Path.expand(output)

-    unless System.get_env("WNH_SCALE_FIXTURE_CONFIRM") == @confirmation do
-      Mix.raise("WNH_SCALE_FIXTURE_CONFIRM must equal #{@confirmation}")
+    expected_confirmation =
+      if action == "append", do: @additive_confirmation, else: @isolated_confirmation
+
+    unless System.get_env("WNH_SCALE_FIXTURE_CONFIRM") == expected_confirmation do
+      Mix.raise("WNH_SCALE_FIXTURE_CONFIRM must equal #{expected_confirmation}")
    end

    unless String.starts_with?(output, "/output/") do
@ -75,6 +80,54 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
    %{database: actual_database, output: output}
  end

+  defp append!(context, request_count, activity_count) do
+    before_totals = discovery_counts()
+    before_synthetic = synthetic_discovery_counts()
+
+    if before_synthetic.help_requests > request_count or
+         before_synthetic.activities > activity_count do
+      Mix.raise(
+        "the development database already contains more synthetic scale rows than requested: " <>
+          "#{before_synthetic.help_requests} requests and #{before_synthetic.activities} activities"
+      )
+    end
+
+    categories = seed_categories!()
+    ensure_viewer!()
+
+    Repo.transaction(
+      fn ->
+        seed_users!()
+        seed_requests!(request_count, categories.help)
+        seed_activities!(activity_count, categories.activity)
+      end,
+      timeout: :infinity
+    )
+
+    Repo.query!(
+      "ANALYZE users, categories, help_requests, activities",
+      [],
+      timeout: :infinity,
+      log: false
+    )
+
+    assert_synthetic_counts!(request_count, activity_count)
+
+    write_summary!(context, "appended", request_count, activity_count, %{
+      expected_kind: "synthetic",
+      totals_before: before_totals,
+      synthetic_before: before_synthetic,
+      synthetic_after: synthetic_discovery_counts()
+    })
+
+    totals = discovery_counts()
+
+    Mix.shell().info(
+      "persistent synthetic scale data is ready alongside existing data in #{context.database}: " <>
+        "#{totals.help_requests} total requests and #{totals.activities} total activities"
+    )
+  end
+
  defp prepare!(context, request_count, activity_count) do
    counts = discovery_counts()

@ -312,6 +365,7 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
        ) AS cities(index, name, longitude, latitude)
        WHERE cities.index = value % 20
      ) AS city
+      ON CONFLICT (id) DO NOTHING
      """,
      [rows, @synthetic_users, category_ids],
      timeout: :infinity,
@ -379,6 +433,7 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
        ) AS cities(index, name, longitude, latitude)
        WHERE cities.index = value % 20
      ) AS city
+      ON CONFLICT (id) DO NOTHING
      """,
      [rows, @synthetic_users, category_ids],
      timeout: :infinity,
@ -465,6 +520,17 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
    end
  end

+  defp assert_synthetic_counts!(request_count, activity_count) do
+    actual = synthetic_discovery_counts()
+
+    unless actual.help_requests == request_count and actual.activities == activity_count do
+      Mix.raise(
+        "expected #{request_count} synthetic requests and #{activity_count} synthetic activities, " <>
+          "found #{actual.help_requests} and #{actual.activities}"
+      )
+    end
+  end
+
  defp discovery_counts do
    %{
      help_requests: scalar!("SELECT count(*) FROM help_requests"),
@ -472,6 +538,21 @@ defmodule Mix.Tasks.Wnh.ScaleFixtures do
    }
  end

+  defp synthetic_discovery_counts do
+    %{
+      help_requests:
+        scalar!(
+          "SELECT count(*) FROM help_requests " <>
+            "WHERE structured_data @> '{\"scale_fixture\": true}'::jsonb"
+        ),
+      activities:
+        scalar!(
+          "SELECT count(*) FROM activities " <>
+            "WHERE structured_data @> '{\"scale_fixture\": true}'::jsonb"
+        )
+    }
+  end
+
  defp write_summary!(context, status, request_count, activity_count, extra) do
    File.mkdir_p!(Path.dirname(context.output))

--- a/scripts/dev-scale-seed.sh
+++ b/scripts/dev-scale-seed.sh
@ -0,0 +1,173 @@
+#!/bin/sh
+set -eu
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+
+if [ ! -f "$ROOT/.env" ]; then
+  echo "Missing $ROOT/.env." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck source=/dev/null
+. "$ROOT/.env"
+set +a
+
+PROJECT=${WNH_DEV_COMPOSE_PROJECT:-who_need_help}
+EXPECTED_DATABASE=${WNH_DEV_SCALE_EXPECTED_DATABASE:-who_need_help}
+REQUESTS=${WNH_DEV_SCALE_REQUESTS:-1000000}
+ACTIVITIES=${WNH_DEV_SCALE_ACTIVITIES:-1000000}
+WEB_CONTAINER=${PROJECT}-web-1
+DB_CONTAINER=${PROJECT}-db-1
+NETWORK=${PROJECT}_internal
+TOOLS_IMAGE=who-need-help:dev-scale-tools
+OUTPUT_DIR="$ROOT/output/dev-scale"
+SUMMARY="$OUTPUT_DIR/summary.json"
+
+container_label() {
+  docker inspect "$1" --format "{{index .Config.Labels \"$2\"}}"
+}
+
+container_env() {
+  docker inspect "$WEB_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' |
+    sed -n "s/^$1=//p"
+}
+
+require_positive_integer() {
+  case "$2" in
+    ''|*[!0-9]*|0)
+      echo "$1 must be a positive integer." >&2
+      exit 2
+      ;;
+  esac
+}
+
+require_positive_integer WNH_DEV_SCALE_REQUESTS "$REQUESTS"
+require_positive_integer WNH_DEV_SCALE_ACTIVITIES "$ACTIVITIES"
+
+for container in "$WEB_CONTAINER" "$DB_CONTAINER"; do
+  if ! docker inspect "$container" >/dev/null 2>&1; then
+    echo "Required development container $container does not exist." >&2
+    exit 1
+  fi
+
+  if [ "$(docker inspect "$container" --format '{{.State.Running}}')" != true ]; then
+    echo "Required development container $container is not running." >&2
+    exit 1
+  fi
+
+  if [ "$(container_label "$container" com.docker.compose.project)" != "$PROJECT" ]; then
+    echo "Container $container does not belong to Compose project $PROJECT." >&2
+    exit 1
+  fi
+done
+
+if [ "$(container_label "$WEB_CONTAINER" com.docker.compose.service)" != web ]; then
+  echo "$WEB_CONTAINER is not the development web service." >&2
+  exit 1
+fi
+
+if [ "$(container_label "$DB_CONTAINER" com.docker.compose.service)" != db ]; then
+  echo "$DB_CONTAINER is not the development database service." >&2
+  exit 1
+fi
+
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+  echo "Development network $NETWORK does not exist." >&2
+  exit 1
+fi
+
+DATABASE_URL=$(container_env DATABASE_URL)
+SECRET_KEY_BASE=$(container_env SECRET_KEY_BASE)
+HANDOVER_SECRET=$(container_env HANDOVER_SECRET)
+RELEASE_COOKIE=$(container_env RELEASE_COOKIE)
+METRICS_TOKEN=$(container_env METRICS_TOKEN)
+PHX_HOST=$(container_env PHX_HOST)
+PHX_SCHEME=$(container_env PHX_SCHEME)
+PHX_URL_PORT=$(container_env PHX_URL_PORT)
+DB_USER=$(
+  docker inspect "$DB_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' |
+    sed -n 's/^POSTGRES_USER=//p'
+)
+DB_NAME=$(
+  docker inspect "$DB_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' |
+    sed -n 's/^POSTGRES_DB=//p'
+)
+
+if [ -z "${DEV_SCALE_VIEWER_PASSWORD:-}" ]; then
+  echo "DEV_SCALE_VIEWER_PASSWORD must be set in the project .env." >&2
+  exit 1
+fi
+
+for pair in \
+  "DATABASE_URL:$DATABASE_URL" \
+  "SECRET_KEY_BASE:$SECRET_KEY_BASE" \
+  "HANDOVER_SECRET:$HANDOVER_SECRET" \
+  "RELEASE_COOKIE:$RELEASE_COOKIE" \
+  "METRICS_TOKEN:$METRICS_TOKEN" \
+  "PHX_HOST:$PHX_HOST" \
+  "PHX_SCHEME:$PHX_SCHEME" \
+  "PHX_URL_PORT:$PHX_URL_PORT"; do
+  name=${pair%%:*}
+  value=${pair#*:}
+
+  if [ -z "$value" ]; then
+    echo "The running development web container is missing $name." >&2
+    exit 1
+  fi
+done
+
+export DATABASE_URL SECRET_KEY_BASE HANDOVER_SECRET RELEASE_COOKIE METRICS_TOKEN
+export PHX_HOST PHX_SCHEME PHX_URL_PORT
+SCALE_VIEWER_PASSWORD=$DEV_SCALE_VIEWER_PASSWORD
+export SCALE_VIEWER_PASSWORD
+
+if [ -z "$DB_USER" ] || [ -z "$DB_NAME" ]; then
+  echo "The running development database container is missing POSTGRES_USER or POSTGRES_DB." >&2
+  exit 1
+fi
+
+actual_database=$(
+  docker exec "$DB_CONTAINER" \
+    psql -U "$DB_USER" -d "$DB_NAME" -Atc "SELECT current_database()"
+)
+
+if [ "$actual_database" != "$EXPECTED_DATABASE" ]; then
+  echo "Refusing dev-scale mutation: expected database $EXPECTED_DATABASE, observed $actual_database." >&2
+  exit 1
+fi
+
+mkdir -p "$OUTPUT_DIR"
+docker build --target load_tools --tag "$TOOLS_IMAGE" "$ROOT"
+
+docker run --rm \
+  --network "$NETWORK" \
+  --env APP_ROLE=migrate \
+  --env DATABASE_URL \
+  --env POOL_SIZE=2 \
+  --env ERL_ZFLAGS='+Q 65536 +S 1:1' \
+  --env SECRET_KEY_BASE \
+  --env HANDOVER_SECRET \
+  --env RELEASE_COOKIE \
+  --env METRICS_TOKEN \
+  --env PHX_HOST \
+  --env PHX_SCHEME \
+  --env PHX_URL_PORT \
+  --env SCALE_VIEWER_PASSWORD \
+  --env EMAIL_FROM_NAME='Who Need Help development scale fixtures' \
+  --env EMAIL_FROM_ADDRESS=dev-scale@example.invalid \
+  --env WNH_SCALE_EXPECTED_DATABASE="$EXPECTED_DATABASE" \
+  --env WNH_SCALE_FIXTURE_CONFIRM=persistent-dev-scale \
+  --volume "$OUTPUT_DIR:/output" \
+  "$TOOLS_IMAGE" \
+  mix wnh.scale_fixtures append \
+    --requests "$REQUESTS" \
+    --activities "$ACTIVITIES" \
+    --output /output/summary.json
+
+docker run --rm \
+  --volume "$OUTPUT_DIR:/output" \
+  alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 \
+  chown -R "$(id -u):$(id -g)" /output
+
+printf 'Development scale summary: %s\n' "$SUMMARY"
--- a/scripts/scale-stack-down.sh
+++ b/scripts/scale-stack-down.sh
@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+# shellcheck source=scripts/scale-common.sh
+. "$ROOT/scripts/scale-common.sh"
+scale_load_environment
+
+scale_compose down
+printf 'Removed %s containers and networks; its PostgreSQL volume was preserved.\n' "$LOAD_PROJECT"