who_need_help/scripts/load-run.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
ENV_FILE="$ROOT/.env.load"
K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669"
PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4"
LABEL=${1:-"run-$(date -u +%Y%m%dT%H%M%SZ)"}
duration_override=${LOAD_DURATION_OVERRIDE:-}

if [[ ! -f "$ENV_FILE" ]]; then
  echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
  exit 1
fi

set -a
# shellcheck source=/dev/null
. "$ENV_FILE"
set +a

duration_source=".env.load"
if [[ -n "$duration_override" ]]; then
  LOAD_DURATION=$duration_override
  duration_source="LOAD_DURATION_OVERRIDE"
fi

for name in LOAD_PROJECT LOAD_HOST LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS \
  LOAD_HTTP_VUS LOAD_WS_VUS \
  LOAD_DURATION LOAD_WS_HOLD_MS LOAD_WS_CONNECT_TIMEOUT_MS \
  LOAD_HTTP_THINK_SECONDS LOAD_AUTH_VUS LOAD_AUTH_WS_TIMEOUT_MS \
  LOAD_AUTH_THINK_SECONDS LOAD_FIXTURE_PASSWORD POSTGRES_DB HTTP_PORT; do
  if [[ -z "${!name:-}" ]]; then
    echo "$name is missing from .env.load" >&2
    exit 1
  fi
done

for name in LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS LOAD_HTTP_VUS LOAD_WS_VUS \
  LOAD_AUTH_VUS; do
  if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then
    echo "$name must be a positive integer." >&2
    exit 1
  fi
done

fixture_count=$((LOAD_HTTP_VUS + LOAD_WS_VUS + LOAD_AUTH_VUS))

if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
  echo "The load profile must not use the staging Compose project." >&2
  exit 1
fi

if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
  echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
  exit 1
fi

compose=(
  docker compose
  --env-file "$ENV_FILE"
  -p "$LOAD_PROJECT"
  -f compose.yaml
  -f compose.load.yaml
)

mapfile -t web_containers < <("${compose[@]}" ps -q web)
mapfile -t worker_containers < <("${compose[@]}" ps -q worker)

if [[ "${#web_containers[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then
  echo "Expected $LOAD_WEB_REPLICAS running web replicas; observed ${#web_containers[@]}." >&2
  exit 1
fi

if [[ "${#worker_containers[@]}" -ne "$LOAD_WORKER_REPLICAS" ]]; then
  echo "Expected $LOAD_WORKER_REPLICAS running worker replicas; observed ${#worker_containers[@]}." >&2
  exit 1
fi

network_id=$(
  docker network ls \
    --filter "label=com.docker.compose.project=$LOAD_PROJECT" \
    --filter "label=com.docker.compose.network=edge" \
    --quiet |
    head -n 1
)

if [[ -z "$network_id" ]]; then
  echo "The isolated load-profile edge network was not found." >&2
  exit 1
fi

internal_network_id=$(
  docker network ls \
    --filter "label=com.docker.compose.project=$LOAD_PROJECT" \
    --filter "label=com.docker.compose.network=internal" \
    --quiet |
    head -n 1
)

if [[ -z "$internal_network_id" ]]; then
  echo "The isolated load-profile internal network was not found." >&2
  exit 1
fi

if ! docker image inspect who-need-help:load-tools >/dev/null 2>&1; then
  echo "The isolated load-tools image is missing. Run scripts/load-stack-up.sh first." >&2
  exit 1
fi

mapfile -t measured_containers < <(
  "${compose[@]}" ps -q web worker db proxy
)

if [[ "${#measured_containers[@]}" -lt 4 ]]; then
  echo "The isolated load stack is incomplete." >&2
  exit 1
fi

output_dir="$ROOT/output/performance/$LABEL"
mkdir -p "$output_dir"
chmod 700 "$ROOT/output" "$ROOT/output/performance" "$output_dir"
run_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
running_marker="$output_dir/.sampling"
resource_log="$output_dir/docker-stats.jsonl"
touch "$running_marker"

snapshot_database() {
  local destination=$1

  # The variables below are intentionally expanded inside the database container.
  # shellcheck disable=SC2016
  "${compose[@]}" exec -T db sh -c \
    'psql --no-psqlrc --set ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
    >"$destination" <<'SQL'
BEGIN READ ONLY;
SELECT 'users' AS table_name, count(*) AS row_count FROM users
UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
UNION ALL SELECT 'help_requests', count(*) FROM help_requests
UNION ALL SELECT 'messages', count(*) FROM messages
UNION ALL SELECT 'categories', count(*) FROM categories
UNION ALL SELECT 'category_proposals', count(*) FROM category_proposals
UNION ALL SELECT 'category_votes', count(*) FROM category_votes
UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
UNION ALL SELECT 'activities', count(*) FROM activities
UNION ALL SELECT 'activity_participants', count(*) FROM activity_participants
UNION ALL SELECT 'activity_messages', count(*) FROM activity_messages
UNION ALL SELECT 'reports', count(*) FROM reports
UNION ALL SELECT 'reviews', count(*) FROM reviews
UNION ALL SELECT 'blocks', count(*) FROM blocks
UNION ALL SELECT 'audit_events', count(*) FROM audit_events
UNION ALL SELECT 'abuse_signals', count(*) FROM abuse_signals
UNION ALL SELECT 'rate_limit_buckets', count(*) FROM rate_limit_buckets
UNION ALL SELECT 'social_identities', count(*) FROM social_identities
UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
ORDER BY table_name;
COMMIT;
SQL
}

snapshot_database_metrics() {
  local destination=$1

  # The variables below are intentionally expanded inside the database container.
  # shellcheck disable=SC2016
  "${compose[@]}" exec -T db sh -c \
    'psql --no-psqlrc --quiet --tuples-only --no-align --set ON_ERROR_STOP=1 \
      --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
    >"$destination" <<'SQL'
BEGIN READ ONLY;
SELECT json_build_object(
  'captured_at', clock_timestamp(),
  'database_size_bytes', pg_database_size(current_database()),
  'stats_reset', stats_reset,
  'numbackends', numbackends,
  'xact_commit', xact_commit,
  'xact_rollback', xact_rollback,
  'blks_read', blks_read,
  'blks_hit', blks_hit,
  'tup_returned', tup_returned,
  'tup_fetched', tup_fetched,
  'tup_inserted', tup_inserted,
  'tup_updated', tup_updated,
  'tup_deleted', tup_deleted,
  'temp_files', temp_files,
  'temp_bytes', temp_bytes,
  'deadlocks', deadlocks,
  'conflicts', conflicts
)
FROM pg_stat_database
WHERE datname = current_database();
COMMIT;
SQL
}

summarize_database_metrics() {
  local before=$1
  local after=$2
  local destination=$3

  jq -n \
    --slurpfile before "$before" \
    --slurpfile after "$after" '
      def counters:
        [
          "database_size_bytes",
          "xact_commit",
          "xact_rollback",
          "blks_read",
          "blks_hit",
          "tup_returned",
          "tup_fetched",
          "tup_inserted",
          "tup_updated",
          "tup_deleted",
          "temp_files",
          "temp_bytes",
          "deadlocks",
          "conflicts"
        ];

      ($before[0]) as $before |
      ($after[0]) as $after |
      {
        schema_version: 1,
        measurement: "read-only pg_stat_database snapshots around the k6 run",
        thresholds_applied: false,
        stats_reset_before: $before.stats_reset,
        stats_reset_after: $after.stats_reset,
        before: $before,
        after: $after,
        delta: reduce counters[] as $key
          ({}; .[$key] = ($after[$key] - $before[$key]))
      }
    ' >"$destination"
}

summarize_resources() {
  docker run --rm \
    --user "$(id -u):$(id -g)" \
    --volume "$ROOT/scripts/summarize-docker-stats.py:/scripts/summarize-docker-stats.py:ro" \
    --volume "$output_dir:/output" \
    "$PYTHON_IMAGE" \
    python /scripts/summarize-docker-stats.py \
      /output/docker-stats.jsonl \
      /output/resource-summary.json
}

run_fixture_tool() {
  local action=$1

  docker run --rm \
    --network "$internal_network_id" \
    --env-file "$ENV_FILE" \
    --env APP_ROLE=migrate \
    --env "WNH_LOAD_EXPECTED_DATABASE=$POSTGRES_DB" \
    --env WNH_LOAD_FIXTURE_CONFIRM=isolated-load-fixtures \
    --env "WNH_LOAD_FIXTURE_RUN_ID=$LABEL" \
    --env "WNH_LOAD_FIXTURE_COUNT=$fixture_count" \
    --env "WNH_LOAD_FIXTURE_PATH=/output/fixtures.json" \
    --volume "$output_dir:/output" \
    who-need-help:load-tools \
    mix wnh.load_fixtures "$action"
}

validate_authenticated_writes() {
  # The run label was restricted to a conservative character set above and is
  # passed as a psql variable, not interpolated into SQL syntax.
  # shellcheck disable=SC2016
  "${compose[@]}" exec -T db sh -c \
    'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
      --set run_id="$1" --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
    sh "$LABEL" >"$output_dir/authenticated-writes.json" <<'SQL'
SELECT json_build_object(
  'messages',
    (SELECT count(*) FROM messages WHERE body LIKE 'load:' || :'run_id' || ':%'),
  'tracking_samples',
    (SELECT coalesce(sum(session.sample_count), 0)
       FROM tracking_sessions AS session
       JOIN users AS actor ON actor.id = session.user_id
      WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
            'wnh-load-' || lower(:'run_id') || '-'),
  'tracking_sessions',
    (SELECT count(*)
       FROM tracking_sessions AS session
       JOIN users AS actor ON actor.id = session.user_id
      WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
            'wnh-load-' || lower(:'run_id') || '-'),
  'tracking_positions',
    (SELECT count(*)
       FROM tracking_positions AS position
       JOIN tracking_sessions AS session ON session.id = position.tracking_session_id
       JOIN users AS actor ON actor.id = session.user_id
      WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
            'wnh-load-' || lower(:'run_id') || '-'),
  'session_tokens',
    (SELECT count(*)
       FROM users_tokens AS token
       JOIN users AS actor ON actor.id = token.user_id
      WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
            'wnh-load-' || lower(:'run_id') || '-')
);
SQL

  jq -e '
    .messages > 0 and
    .tracking_samples > 0 and
    .tracking_sessions > 0 and
    .tracking_positions == 0 and
    .session_tokens > 0
  ' "$output_dir/authenticated-writes.json" >/dev/null
}

sample_resources() {
  while [[ -e "$running_marker" ]]; do
    observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    docker stats --no-stream --format '{{json .}}' "${measured_containers[@]}" |
      jq -c --arg observed_at "$observed_at" '. + {ObservedAt: $observed_at}' \
        >>"$resource_log"
    sleep 1
  done
}

cleanup_sampler() {
  rm -f "$running_marker"

  if [[ -n "${sampler_pid:-}" ]]; then
    wait "$sampler_pid" 2>/dev/null || true
  fi
}

fixtures_prepared=false

cleanup_on_exit() {
  local status=$?
  local cleanup_status=0

  trap - EXIT HUP INT TERM
  cleanup_sampler

  if [[ "$fixtures_prepared" == true ]]; then
    set +e
    run_fixture_tool cleanup >>"$output_dir/fixture-cleanup.log" 2>&1
    cleanup_status=$?
    set -e
  fi

  if [[ "$cleanup_status" -ne 0 ]]; then
    echo "Fixture cleanup failed; inspect $output_dir/fixture-cleanup.log." >&2
    status=1
  fi

  exit "$status"
}

trap cleanup_on_exit EXIT HUP INT TERM

{
  printf 'observed_at=%s\n' "$run_started_at"
  printf 'k6_image=%s\n' "$K6_IMAGE"
  printf 'resource_summarizer_image=%s\n' "$PYTHON_IMAGE"
  printf 'load_project=%s\n' "$LOAD_PROJECT"
  printf 'web_replicas=%s\n' "$LOAD_WEB_REPLICAS"
  printf 'worker_replicas=%s\n' "$LOAD_WORKER_REPLICAS"
  printf 'http_vus=%s\n' "$LOAD_HTTP_VUS"
  printf 'websocket_vus=%s\n' "$LOAD_WS_VUS"
  printf 'duration=%s\n' "$LOAD_DURATION"
  printf 'duration_source=%s\n' "$duration_source"
  printf 'websocket_hold_ms=%s\n' "$LOAD_WS_HOLD_MS"
  printf 'websocket_connect_timeout_ms=%s\n' "$LOAD_WS_CONNECT_TIMEOUT_MS"
  printf 'http_think_seconds=%s\n' "$LOAD_HTTP_THINK_SECONDS"
  printf 'authenticated_vus=%s\n' "$LOAD_AUTH_VUS"
  printf 'fixture_count=%s\n' "$fixture_count"
  printf 'authenticated_websocket_timeout_ms=%s\n' "$LOAD_AUTH_WS_TIMEOUT_MS"
  printf 'authenticated_think_seconds=%s\n' "$LOAD_AUTH_THINK_SECONDS"
  docker info --format 'docker_cpus={{.NCPU}} docker_memory_bytes={{.MemTotal}} docker_server={{.ServerVersion}}'
  docker compose version
  uname -a
} >"$output_dir/environment.txt"

snapshot_database "$output_dir/database-pre-fixtures.txt"
run_fixture_tool prepare >"$output_dir/fixture-prepare.log" 2>&1
fixtures_prepared=true
snapshot_database "$output_dir/database-before-load.txt"
snapshot_database_metrics "$output_dir/database-metrics-before-load.json"
sample_resources &
sampler_pid=$!

set +e
docker run --rm \
  --user "$(id -u):$(id -g)" \
  --network "$network_id" \
  --volume "$ROOT/load/k6:/scripts:ro" \
  --volume "$output_dir:/fixtures:ro" \
  --volume "$output_dir:/output" \
  --env "BASE_URL=https://$LOAD_HOST" \
  --env "PUBLIC_ORIGIN=https://$LOAD_HOST" \
  --env "HTTP_VUS=$LOAD_HTTP_VUS" \
  --env "WS_VUS=$LOAD_WS_VUS" \
  --env "DURATION=$LOAD_DURATION" \
  --env "WS_HOLD_MS=$LOAD_WS_HOLD_MS" \
  --env "WS_CONNECT_TIMEOUT_MS=$LOAD_WS_CONNECT_TIMEOUT_MS" \
  --env "HTTP_THINK_SECONDS=$LOAD_HTTP_THINK_SECONDS" \
  --env "AUTH_VUS=$LOAD_AUTH_VUS" \
  --env "AUTH_WS_TIMEOUT_MS=$LOAD_AUTH_WS_TIMEOUT_MS" \
  --env "AUTH_THINK_SECONDS=$LOAD_AUTH_THINK_SECONDS" \
  --env "FIXTURE_PASSWORD=$LOAD_FIXTURE_PASSWORD" \
  --env "FIXTURE_RUN_ID=$LABEL" \
  "$K6_IMAGE" run \
  --no-usage-report \
  --summary-mode=full \
  --summary-export=/output/k6-summary.json \
  --new-machine-readable-summary \
  /scripts/public-and-websocket.js 2>&1 |
  tee "$output_dir/k6.log"
k6_status=${PIPESTATUS[0]}
set -e

cleanup_sampler
sampler_pid=
summarize_resources

snapshot_database "$output_dir/database-after-load.txt"
snapshot_database_metrics "$output_dir/database-metrics-after-load.json"
summarize_database_metrics \
  "$output_dir/database-metrics-before-load.json" \
  "$output_dir/database-metrics-after-load.json" \
  "$output_dir/database-metrics-summary.json"
diff -u "$output_dir/database-before-load.txt" "$output_dir/database-after-load.txt" \
  >"$output_dir/database-load-diff.txt" || true
validate_authenticated_writes

COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-realtime-cluster.sh" compose \
  >"$output_dir/pubsub-probe.txt"
curl --fail --silent --show-error \
  --header "Host: $LOAD_HOST" \
  "http://localhost:$HTTP_PORT/healthz/ready" \
  >"$output_dir/readiness-after.json"
"${compose[@]}" ps -a >"$output_dir/compose-after.txt"
"${compose[@]}" logs --since "$run_started_at" proxy web worker \
  >"$output_dir/application.log" 2>&1

if ! jq -e '
  def metric($name):
    ([.results.metrics[] | select(.name == $name) | .values][0] // {});

  (.results.checks.metrics[] |
    select(.name == "checks_failed") |
    .values.matches) == 0 and
  metric("http_req_failed").matches == 0 and
  metric("wnh_websocket_opened").count > 0 and
  metric("wnh_websocket_opened").count ==
    metric("wnh_websocket_heartbeat_replies").count and
  (metric("wnh_websocket_errors").count // 0) == 0 and
  metric("wnh_authenticated_logins").count > 0 and
  metric("wnh_authenticated_pages").count > 0 and
  metric("wnh_tracking_updates").count > 0 and
  metric("wnh_liveview_joins").count > 0 and
  metric("wnh_liveview_joins").count ==
    metric("wnh_liveview_tracking_starts").count and
  metric("wnh_liveview_joins").count == metric("wnh_tracking_updates").count and
  metric("wnh_liveview_joins").count == metric("wnh_liveview_messages").count and
  metric("wnh_liveview_joins").count ==
    metric("wnh_liveview_tracking_stops").count and
  (metric("wnh_authenticated_errors").count // 0) == 0
' "$output_dir/k6-summary.json" >/dev/null; then
  echo "Functional load checks failed; evidence is in $output_dir." >&2
  exit 1
fi

if [[ "$k6_status" -ne 0 ]]; then
  echo "k6 exited with status $k6_status; evidence is in $output_dir." >&2
  exit "$k6_status"
fi

run_fixture_tool cleanup >"$output_dir/fixture-cleanup.log" 2>&1
fixtures_prepared=false
snapshot_database "$output_dir/database-after-cleanup.txt"
snapshot_database_metrics "$output_dir/database-metrics-after-cleanup.json"

if ! diff -u "$output_dir/database-pre-fixtures.txt" \
  "$output_dir/database-after-cleanup.txt" >"$output_dir/database-cleanup-diff.txt"; then
  echo "Fixture cleanup did not restore the pre-run table counts; evidence is in $output_dir." >&2
  exit 1
fi

trap - EXIT HUP INT TERM
printf 'Load evidence: %s\n' "$output_dir"