#!/usr/bin/env bash set -euo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) ENV_FILE="$ROOT/.env.load" K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669" PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4" LABEL=${1:-"run-$(date -u +%Y%m%dT%H%M%SZ)"} duration_override=${LOAD_DURATION_OVERRIDE:-} if [[ ! -f "$ENV_FILE" ]]; then echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2 exit 1 fi set -a # shellcheck source=/dev/null . "$ENV_FILE" set +a duration_source=".env.load" if [[ -n "$duration_override" ]]; then LOAD_DURATION=$duration_override duration_source="LOAD_DURATION_OVERRIDE" fi for name in LOAD_PROJECT LOAD_HOST LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS \ LOAD_HTTP_VUS LOAD_WS_VUS \ LOAD_DURATION LOAD_WS_HOLD_MS LOAD_WS_CONNECT_TIMEOUT_MS \ LOAD_HTTP_THINK_SECONDS LOAD_AUTH_VUS LOAD_AUTH_WS_TIMEOUT_MS \ LOAD_AUTH_THINK_SECONDS LOAD_FIXTURE_PASSWORD POSTGRES_DB HTTP_PORT; do if [[ -z "${!name:-}" ]]; then echo "$name is missing from .env.load" >&2 exit 1 fi done for name in LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS LOAD_HTTP_VUS LOAD_WS_VUS \ LOAD_AUTH_VUS; do if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then echo "$name must be a positive integer." >&2 exit 1 fi done fixture_count=$((LOAD_HTTP_VUS + LOAD_WS_VUS + LOAD_AUTH_VUS)) if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then echo "The load profile must not use the staging Compose project." >&2 exit 1 fi if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2 exit 1 fi compose=( docker compose --env-file "$ENV_FILE" -p "$LOAD_PROJECT" -f compose.yaml -f compose.load.yaml ) mapfile -t web_containers < <("${compose[@]}" ps -q web) mapfile -t worker_containers < <("${compose[@]}" ps -q worker) if [[ "${#web_containers[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then echo "Expected $LOAD_WEB_REPLICAS running web replicas; observed ${#web_containers[@]}." >&2 exit 1 fi if [[ "${#worker_containers[@]}" -ne "$LOAD_WORKER_REPLICAS" ]]; then echo "Expected $LOAD_WORKER_REPLICAS running worker replicas; observed ${#worker_containers[@]}." >&2 exit 1 fi network_id=$( docker network ls \ --filter "label=com.docker.compose.project=$LOAD_PROJECT" \ --filter "label=com.docker.compose.network=edge" \ --quiet | head -n 1 ) if [[ -z "$network_id" ]]; then echo "The isolated load-profile edge network was not found." >&2 exit 1 fi internal_network_id=$( docker network ls \ --filter "label=com.docker.compose.project=$LOAD_PROJECT" \ --filter "label=com.docker.compose.network=internal" \ --quiet | head -n 1 ) if [[ -z "$internal_network_id" ]]; then echo "The isolated load-profile internal network was not found." >&2 exit 1 fi if ! docker image inspect who-need-help:load-tools >/dev/null 2>&1; then echo "The isolated load-tools image is missing. Run scripts/load-stack-up.sh first." >&2 exit 1 fi mapfile -t measured_containers < <( "${compose[@]}" ps -q web worker db proxy ) if [[ "${#measured_containers[@]}" -lt 4 ]]; then echo "The isolated load stack is incomplete." >&2 exit 1 fi output_dir="$ROOT/output/performance/$LABEL" mkdir -p "$output_dir" chmod 700 "$ROOT/output" "$ROOT/output/performance" "$output_dir" run_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) running_marker="$output_dir/.sampling" resource_log="$output_dir/docker-stats.jsonl" touch "$running_marker" snapshot_database() { local destination=$1 # The variables below are intentionally expanded inside the database container. # shellcheck disable=SC2016 "${compose[@]}" exec -T db sh -c \ 'psql --no-psqlrc --set ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \ >"$destination" <<'SQL' BEGIN READ ONLY; SELECT 'users' AS table_name, count(*) AS row_count FROM users UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens UNION ALL SELECT 'help_requests', count(*) FROM help_requests UNION ALL SELECT 'messages', count(*) FROM messages UNION ALL SELECT 'categories', count(*) FROM categories UNION ALL SELECT 'category_proposals', count(*) FROM category_proposals UNION ALL SELECT 'category_votes', count(*) FROM category_votes UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments UNION ALL SELECT 'activities', count(*) FROM activities UNION ALL SELECT 'activity_participants', count(*) FROM activity_participants UNION ALL SELECT 'activity_messages', count(*) FROM activity_messages UNION ALL SELECT 'reports', count(*) FROM reports UNION ALL SELECT 'reviews', count(*) FROM reviews UNION ALL SELECT 'blocks', count(*) FROM blocks UNION ALL SELECT 'audit_events', count(*) FROM audit_events UNION ALL SELECT 'abuse_signals', count(*) FROM abuse_signals UNION ALL SELECT 'rate_limit_buckets', count(*) FROM rate_limit_buckets UNION ALL SELECT 'social_identities', count(*) FROM social_identities UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions ORDER BY table_name; COMMIT; SQL } snapshot_database_metrics() { local destination=$1 # The variables below are intentionally expanded inside the database container. # shellcheck disable=SC2016 "${compose[@]}" exec -T db sh -c \ 'psql --no-psqlrc --quiet --tuples-only --no-align --set ON_ERROR_STOP=1 \ --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \ >"$destination" <<'SQL' BEGIN READ ONLY; SELECT json_build_object( 'captured_at', clock_timestamp(), 'database_size_bytes', pg_database_size(current_database()), 'stats_reset', stats_reset, 'numbackends', numbackends, 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, 'blks_read', blks_read, 'blks_hit', blks_hit, 'tup_returned', tup_returned, 'tup_fetched', tup_fetched, 'tup_inserted', tup_inserted, 'tup_updated', tup_updated, 'tup_deleted', tup_deleted, 'temp_files', temp_files, 'temp_bytes', temp_bytes, 'deadlocks', deadlocks, 'conflicts', conflicts ) FROM pg_stat_database WHERE datname = current_database(); COMMIT; SQL } summarize_database_metrics() { local before=$1 local after=$2 local destination=$3 jq -n \ --slurpfile before "$before" \ --slurpfile after "$after" ' def counters: [ "database_size_bytes", "xact_commit", "xact_rollback", "blks_read", "blks_hit", "tup_returned", "tup_fetched", "tup_inserted", "tup_updated", "tup_deleted", "temp_files", "temp_bytes", "deadlocks", "conflicts" ]; ($before[0]) as $before | ($after[0]) as $after | { schema_version: 1, measurement: "read-only pg_stat_database snapshots around the k6 run", thresholds_applied: false, stats_reset_before: $before.stats_reset, stats_reset_after: $after.stats_reset, before: $before, after: $after, delta: reduce counters[] as $key ({}; .[$key] = ($after[$key] - $before[$key])) } ' >"$destination" } summarize_resources() { docker run --rm \ --user "$(id -u):$(id -g)" \ --volume "$ROOT/scripts/summarize-docker-stats.py:/scripts/summarize-docker-stats.py:ro" \ --volume "$output_dir:/output" \ "$PYTHON_IMAGE" \ python /scripts/summarize-docker-stats.py \ /output/docker-stats.jsonl \ /output/resource-summary.json } run_fixture_tool() { local action=$1 docker run --rm \ --network "$internal_network_id" \ --env-file "$ENV_FILE" \ --env APP_ROLE=migrate \ --env "WNH_LOAD_EXPECTED_DATABASE=$POSTGRES_DB" \ --env WNH_LOAD_FIXTURE_CONFIRM=isolated-load-fixtures \ --env "WNH_LOAD_FIXTURE_RUN_ID=$LABEL" \ --env "WNH_LOAD_FIXTURE_COUNT=$fixture_count" \ --env "WNH_LOAD_FIXTURE_PATH=/output/fixtures.json" \ --volume "$output_dir:/output" \ who-need-help:load-tools \ mix wnh.load_fixtures "$action" } validate_authenticated_writes() { # The run label was restricted to a conservative character set above and is # passed as a psql variable, not interpolated into SQL syntax. # shellcheck disable=SC2016 "${compose[@]}" exec -T db sh -c \ 'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \ --set run_id="$1" --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \ sh "$LABEL" >"$output_dir/authenticated-writes.json" <<'SQL' SELECT json_build_object( 'messages', (SELECT count(*) FROM messages WHERE body LIKE 'load:' || :'run_id' || ':%'), 'tracking_samples', (SELECT coalesce(sum(session.sample_count), 0) FROM tracking_sessions AS session JOIN users AS actor ON actor.id = session.user_id WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) = 'wnh-load-' || lower(:'run_id') || '-'), 'tracking_sessions', (SELECT count(*) FROM tracking_sessions AS session JOIN users AS actor ON actor.id = session.user_id WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) = 'wnh-load-' || lower(:'run_id') || '-'), 'tracking_positions', (SELECT count(*) FROM tracking_positions AS position JOIN tracking_sessions AS session ON session.id = position.tracking_session_id JOIN users AS actor ON actor.id = session.user_id WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) = 'wnh-load-' || lower(:'run_id') || '-'), 'session_tokens', (SELECT count(*) FROM users_tokens AS token JOIN users AS actor ON actor.id = token.user_id WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) = 'wnh-load-' || lower(:'run_id') || '-') ); SQL jq -e ' .messages > 0 and .tracking_samples > 0 and .tracking_sessions > 0 and .tracking_positions == 0 and .session_tokens > 0 ' "$output_dir/authenticated-writes.json" >/dev/null } sample_resources() { while [[ -e "$running_marker" ]]; do observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) docker stats --no-stream --format '{{json .}}' "${measured_containers[@]}" | jq -c --arg observed_at "$observed_at" '. + {ObservedAt: $observed_at}' \ >>"$resource_log" sleep 1 done } cleanup_sampler() { rm -f "$running_marker" if [[ -n "${sampler_pid:-}" ]]; then wait "$sampler_pid" 2>/dev/null || true fi } fixtures_prepared=false cleanup_on_exit() { local status=$? local cleanup_status=0 trap - EXIT HUP INT TERM cleanup_sampler if [[ "$fixtures_prepared" == true ]]; then set +e run_fixture_tool cleanup >>"$output_dir/fixture-cleanup.log" 2>&1 cleanup_status=$? set -e fi if [[ "$cleanup_status" -ne 0 ]]; then echo "Fixture cleanup failed; inspect $output_dir/fixture-cleanup.log." >&2 status=1 fi exit "$status" } trap cleanup_on_exit EXIT HUP INT TERM { printf 'observed_at=%s\n' "$run_started_at" printf 'k6_image=%s\n' "$K6_IMAGE" printf 'resource_summarizer_image=%s\n' "$PYTHON_IMAGE" printf 'load_project=%s\n' "$LOAD_PROJECT" printf 'web_replicas=%s\n' "$LOAD_WEB_REPLICAS" printf 'worker_replicas=%s\n' "$LOAD_WORKER_REPLICAS" printf 'http_vus=%s\n' "$LOAD_HTTP_VUS" printf 'websocket_vus=%s\n' "$LOAD_WS_VUS" printf 'duration=%s\n' "$LOAD_DURATION" printf 'duration_source=%s\n' "$duration_source" printf 'websocket_hold_ms=%s\n' "$LOAD_WS_HOLD_MS" printf 'websocket_connect_timeout_ms=%s\n' "$LOAD_WS_CONNECT_TIMEOUT_MS" printf 'http_think_seconds=%s\n' "$LOAD_HTTP_THINK_SECONDS" printf 'authenticated_vus=%s\n' "$LOAD_AUTH_VUS" printf 'fixture_count=%s\n' "$fixture_count" printf 'authenticated_websocket_timeout_ms=%s\n' "$LOAD_AUTH_WS_TIMEOUT_MS" printf 'authenticated_think_seconds=%s\n' "$LOAD_AUTH_THINK_SECONDS" docker info --format 'docker_cpus={{.NCPU}} docker_memory_bytes={{.MemTotal}} docker_server={{.ServerVersion}}' docker compose version uname -a } >"$output_dir/environment.txt" snapshot_database "$output_dir/database-pre-fixtures.txt" run_fixture_tool prepare >"$output_dir/fixture-prepare.log" 2>&1 fixtures_prepared=true snapshot_database "$output_dir/database-before-load.txt" snapshot_database_metrics "$output_dir/database-metrics-before-load.json" sample_resources & sampler_pid=$! set +e docker run --rm \ --user "$(id -u):$(id -g)" \ --network "$network_id" \ --volume "$ROOT/load/k6:/scripts:ro" \ --volume "$output_dir:/fixtures:ro" \ --volume "$output_dir:/output" \ --env "BASE_URL=https://$LOAD_HOST" \ --env "PUBLIC_ORIGIN=https://$LOAD_HOST" \ --env "HTTP_VUS=$LOAD_HTTP_VUS" \ --env "WS_VUS=$LOAD_WS_VUS" \ --env "DURATION=$LOAD_DURATION" \ --env "WS_HOLD_MS=$LOAD_WS_HOLD_MS" \ --env "WS_CONNECT_TIMEOUT_MS=$LOAD_WS_CONNECT_TIMEOUT_MS" \ --env "HTTP_THINK_SECONDS=$LOAD_HTTP_THINK_SECONDS" \ --env "AUTH_VUS=$LOAD_AUTH_VUS" \ --env "AUTH_WS_TIMEOUT_MS=$LOAD_AUTH_WS_TIMEOUT_MS" \ --env "AUTH_THINK_SECONDS=$LOAD_AUTH_THINK_SECONDS" \ --env "FIXTURE_PASSWORD=$LOAD_FIXTURE_PASSWORD" \ --env "FIXTURE_RUN_ID=$LABEL" \ "$K6_IMAGE" run \ --no-usage-report \ --summary-mode=full \ --summary-export=/output/k6-summary.json \ --new-machine-readable-summary \ /scripts/public-and-websocket.js 2>&1 | tee "$output_dir/k6.log" k6_status=${PIPESTATUS[0]} set -e cleanup_sampler sampler_pid= summarize_resources snapshot_database "$output_dir/database-after-load.txt" snapshot_database_metrics "$output_dir/database-metrics-after-load.json" summarize_database_metrics \ "$output_dir/database-metrics-before-load.json" \ "$output_dir/database-metrics-after-load.json" \ "$output_dir/database-metrics-summary.json" diff -u "$output_dir/database-before-load.txt" "$output_dir/database-after-load.txt" \ >"$output_dir/database-load-diff.txt" || true validate_authenticated_writes COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-realtime-cluster.sh" compose \ >"$output_dir/pubsub-probe.txt" curl --fail --silent --show-error \ --header "Host: $LOAD_HOST" \ "http://localhost:$HTTP_PORT/healthz/ready" \ >"$output_dir/readiness-after.json" "${compose[@]}" ps -a >"$output_dir/compose-after.txt" "${compose[@]}" logs --since "$run_started_at" proxy web worker \ >"$output_dir/application.log" 2>&1 if ! jq -e ' def metric($name): ([.results.metrics[] | select(.name == $name) | .values][0] // {}); (.results.checks.metrics[] | select(.name == "checks_failed") | .values.matches) == 0 and metric("http_req_failed").matches == 0 and metric("wnh_websocket_opened").count > 0 and metric("wnh_websocket_opened").count == metric("wnh_websocket_heartbeat_replies").count and (metric("wnh_websocket_errors").count // 0) == 0 and metric("wnh_authenticated_logins").count > 0 and metric("wnh_authenticated_pages").count > 0 and metric("wnh_tracking_updates").count > 0 and metric("wnh_liveview_joins").count > 0 and metric("wnh_liveview_joins").count == metric("wnh_liveview_tracking_starts").count and metric("wnh_liveview_joins").count == metric("wnh_tracking_updates").count and metric("wnh_liveview_joins").count == metric("wnh_liveview_messages").count and metric("wnh_liveview_joins").count == metric("wnh_liveview_tracking_stops").count and (metric("wnh_authenticated_errors").count // 0) == 0 ' "$output_dir/k6-summary.json" >/dev/null; then echo "Functional load checks failed; evidence is in $output_dir." >&2 exit 1 fi if [[ "$k6_status" -ne 0 ]]; then echo "k6 exited with status $k6_status; evidence is in $output_dir." >&2 exit "$k6_status" fi run_fixture_tool cleanup >"$output_dir/fixture-cleanup.log" 2>&1 fixtures_prepared=false snapshot_database "$output_dir/database-after-cleanup.txt" snapshot_database_metrics "$output_dir/database-metrics-after-cleanup.json" if ! diff -u "$output_dir/database-pre-fixtures.txt" \ "$output_dir/database-after-cleanup.txt" >"$output_dir/database-cleanup-diff.txt"; then echo "Fixture cleanup did not restore the pre-run table counts; evidence is in $output_dir." >&2 exit 1 fi trap - EXIT HUP INT TERM printf 'Load evidence: %s\n' "$output_dir"