879 lines
28 KiB
Bash
Executable File
879 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
|
ENV_FILE="$ROOT/.env.load"
|
|
K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669"
|
|
PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4"
|
|
LABEL=${1:-"run-$(date -u +%Y%m%dT%H%M%SZ)"}
|
|
duration_override=${LOAD_DURATION_OVERRIDE:-}
|
|
|
|
if [[ ! -f "$ENV_FILE" ]]; then
|
|
echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
set -a
|
|
# shellcheck source=/dev/null
|
|
. "$ENV_FILE"
|
|
set +a
|
|
|
|
duration_source=".env.load"
|
|
if [[ -n "$duration_override" ]]; then
|
|
LOAD_DURATION=$duration_override
|
|
duration_source="LOAD_DURATION_OVERRIDE"
|
|
fi
|
|
|
|
for name in LOAD_PROJECT LOAD_HOST LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS \
|
|
LOAD_HTTP_VUS LOAD_WS_VUS \
|
|
LOAD_DURATION LOAD_WS_HOLD_MS LOAD_WS_CONNECT_TIMEOUT_MS \
|
|
LOAD_HTTP_THINK_SECONDS LOAD_AUTH_VUS LOAD_AUTH_WS_TIMEOUT_MS \
|
|
LOAD_AUTH_THINK_SECONDS LOAD_FIXTURE_PASSWORD POSTGRES_DB HTTP_PORT; do
|
|
if [[ -z "${!name:-}" ]]; then
|
|
echo "$name is missing from .env.load" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
for name in LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS LOAD_HTTP_VUS LOAD_WS_VUS \
|
|
LOAD_AUTH_VUS; do
|
|
if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "$name must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
fixture_count=$((LOAD_HTTP_VUS + LOAD_WS_VUS + LOAD_AUTH_VUS))
|
|
|
|
if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
|
|
echo "The load profile must not use the staging Compose project." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
|
|
echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
|
|
exit 1
|
|
fi
|
|
|
|
compose=(
|
|
docker compose
|
|
--env-file "$ENV_FILE"
|
|
-p "$LOAD_PROJECT"
|
|
-f compose.yaml
|
|
-f compose.load.yaml
|
|
)
|
|
|
|
mapfile -t web_containers < <("${compose[@]}" ps -q web)
|
|
mapfile -t worker_containers < <("${compose[@]}" ps -q worker)
|
|
|
|
if [[ "${#web_containers[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then
|
|
echo "Expected $LOAD_WEB_REPLICAS running web replicas; observed ${#web_containers[@]}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${#worker_containers[@]}" -ne "$LOAD_WORKER_REPLICAS" ]]; then
|
|
echo "Expected $LOAD_WORKER_REPLICAS running worker replicas; observed ${#worker_containers[@]}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
network_id=$(
|
|
docker network ls \
|
|
--filter "label=com.docker.compose.project=$LOAD_PROJECT" \
|
|
--filter "label=com.docker.compose.network=edge" \
|
|
--quiet |
|
|
head -n 1
|
|
)
|
|
|
|
if [[ -z "$network_id" ]]; then
|
|
echo "The isolated load-profile edge network was not found." >&2
|
|
exit 1
|
|
fi
|
|
|
|
internal_network_id=$(
|
|
docker network ls \
|
|
--filter "label=com.docker.compose.project=$LOAD_PROJECT" \
|
|
--filter "label=com.docker.compose.network=internal" \
|
|
--quiet |
|
|
head -n 1
|
|
)
|
|
|
|
if [[ -z "$internal_network_id" ]]; then
|
|
echo "The isolated load-profile internal network was not found." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! docker image inspect who-need-help:load-tools >/dev/null 2>&1; then
|
|
echo "The isolated load-tools image is missing. Run scripts/load-stack-up.sh first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
mapfile -t measured_containers < <(
|
|
"${compose[@]}" ps -q web worker db proxy
|
|
)
|
|
|
|
if [[ "${#measured_containers[@]}" -lt 4 ]]; then
|
|
echo "The isolated load stack is incomplete." >&2
|
|
exit 1
|
|
fi
|
|
|
|
output_dir="$ROOT/output/performance/$LABEL"
|
|
mkdir -p "$output_dir"
|
|
chmod 700 "$ROOT/output" "$ROOT/output/performance" "$output_dir"
|
|
run_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
running_marker="$output_dir/.sampling"
|
|
resource_log="$output_dir/docker-stats.jsonl"
|
|
connection_log="$output_dir/database-connections.jsonl"
|
|
touch "$running_marker"
|
|
|
|
database_psql() {
|
|
# The variables below are intentionally expanded inside the database container.
|
|
# shellcheck disable=SC2016
|
|
"${compose[@]}" exec -T db sh -c \
|
|
'psql --no-psqlrc --quiet --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
|
--username "$POSTGRES_USER" --dbname "$POSTGRES_DB"'
|
|
}
|
|
|
|
prepare_statement_stats() {
|
|
local preloaded
|
|
preloaded=$(database_psql <<'SQL'
|
|
SELECT current_setting('shared_preload_libraries')
|
|
~ '(^|,)pg_stat_statements(,|$)';
|
|
SQL
|
|
)
|
|
|
|
if [[ "$preloaded" != "t" ]]; then
|
|
echo "pg_stat_statements is not present in shared_preload_libraries." >&2
|
|
exit 1
|
|
fi
|
|
|
|
database_psql >"$output_dir/statement-statistics-setup.txt" <<'SQL'
|
|
SELECT current_setting('shared_preload_libraries');
|
|
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
|
SQL
|
|
}
|
|
|
|
reset_statement_stats() {
|
|
database_psql >"$output_dir/statement-statistics-reset.txt" <<'SQL'
|
|
SELECT pg_stat_statements_reset();
|
|
SQL
|
|
}
|
|
|
|
snapshot_statement_stats() {
|
|
local destination=$1
|
|
|
|
database_psql >"$destination" <<'SQL'
|
|
WITH statements AS (
|
|
SELECT
|
|
queryid::text AS query_id,
|
|
regexp_replace(query, '[[:space:]]+', ' ', 'g') AS query,
|
|
calls,
|
|
rows,
|
|
round(total_exec_time::numeric, 3) AS total_exec_time_ms,
|
|
round(mean_exec_time::numeric, 3) AS mean_exec_time_ms,
|
|
round(max_exec_time::numeric, 3) AS max_exec_time_ms,
|
|
shared_blks_hit,
|
|
shared_blks_read,
|
|
shared_blks_dirtied,
|
|
shared_blks_written,
|
|
temp_blks_read,
|
|
temp_blks_written,
|
|
wal_records,
|
|
wal_fpi,
|
|
wal_bytes
|
|
FROM pg_stat_statements
|
|
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
|
|
AND query NOT LIKE '%pg_stat_statements%'
|
|
AND query NOT LIKE '%pg_stat_activity%'
|
|
AND query NOT LIKE '%pg_stat_database%'
|
|
),
|
|
top_total AS (
|
|
SELECT *
|
|
FROM statements
|
|
ORDER BY total_exec_time_ms DESC, query_id
|
|
LIMIT 25
|
|
),
|
|
top_mean AS (
|
|
SELECT *
|
|
FROM statements
|
|
ORDER BY mean_exec_time_ms DESC, query_id
|
|
LIMIT 25
|
|
),
|
|
top_max AS (
|
|
SELECT *
|
|
FROM statements
|
|
ORDER BY max_exec_time_ms DESC, query_id
|
|
LIMIT 25
|
|
)
|
|
SELECT jsonb_pretty(
|
|
jsonb_build_object(
|
|
'captured_at', clock_timestamp(),
|
|
'measurement', 'pg_stat_statements since the pre-load reset',
|
|
'statement_count', (SELECT count(*) FROM statements),
|
|
'statistics_info', (SELECT to_jsonb(info) FROM pg_stat_statements_info AS info),
|
|
'top_by_total_execution_time',
|
|
(SELECT coalesce(
|
|
jsonb_agg(
|
|
to_jsonb(item)
|
|
ORDER BY total_exec_time_ms DESC, query_id
|
|
),
|
|
'[]'::jsonb
|
|
) FROM top_total AS item),
|
|
'top_by_mean_execution_time',
|
|
(SELECT coalesce(
|
|
jsonb_agg(
|
|
to_jsonb(item)
|
|
ORDER BY mean_exec_time_ms DESC, query_id
|
|
),
|
|
'[]'::jsonb
|
|
) FROM top_mean AS item),
|
|
'top_by_max_execution_time',
|
|
(SELECT coalesce(
|
|
jsonb_agg(
|
|
to_jsonb(item)
|
|
ORDER BY max_exec_time_ms DESC, query_id
|
|
),
|
|
'[]'::jsonb
|
|
) FROM top_max AS item)
|
|
)
|
|
);
|
|
SQL
|
|
}
|
|
|
|
snapshot_database() {
|
|
local destination=$1
|
|
|
|
# The variables below are intentionally expanded inside the database container.
|
|
# shellcheck disable=SC2016
|
|
"${compose[@]}" exec -T db sh -c \
|
|
'psql --no-psqlrc --set ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
|
|
>"$destination" <<'SQL'
|
|
BEGIN READ ONLY;
|
|
SELECT 'users' AS table_name, count(*) AS row_count FROM users
|
|
UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
|
|
UNION ALL SELECT 'help_requests', count(*) FROM help_requests
|
|
UNION ALL SELECT 'messages', count(*) FROM messages
|
|
UNION ALL SELECT 'categories', count(*) FROM categories
|
|
UNION ALL SELECT 'category_proposals', count(*) FROM category_proposals
|
|
UNION ALL SELECT 'category_votes', count(*) FROM category_votes
|
|
UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
|
|
UNION ALL SELECT 'activities', count(*) FROM activities
|
|
UNION ALL SELECT 'activity_participants', count(*) FROM activity_participants
|
|
UNION ALL SELECT 'activity_messages', count(*) FROM activity_messages
|
|
UNION ALL SELECT 'reports', count(*) FROM reports
|
|
UNION ALL SELECT 'reviews', count(*) FROM reviews
|
|
UNION ALL SELECT 'blocks', count(*) FROM blocks
|
|
UNION ALL SELECT 'audit_events', count(*) FROM audit_events
|
|
UNION ALL SELECT 'abuse_signals', count(*) FROM abuse_signals
|
|
UNION ALL SELECT 'rate_limit_buckets', count(*) FROM rate_limit_buckets
|
|
UNION ALL SELECT 'social_identities', count(*) FROM social_identities
|
|
UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
|
|
UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
|
|
ORDER BY table_name;
|
|
COMMIT;
|
|
SQL
|
|
}
|
|
|
|
snapshot_database_metrics() {
|
|
local destination=$1
|
|
|
|
# The variables below are intentionally expanded inside the database container.
|
|
# shellcheck disable=SC2016
|
|
"${compose[@]}" exec -T db sh -c \
|
|
'psql --no-psqlrc --quiet --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
|
--username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
|
|
>"$destination" <<'SQL'
|
|
BEGIN READ ONLY;
|
|
SELECT json_build_object(
|
|
'captured_at', clock_timestamp(),
|
|
'database_size_bytes', pg_database_size(current_database()),
|
|
'stats_reset', stats_reset,
|
|
'numbackends', numbackends,
|
|
'xact_commit', xact_commit,
|
|
'xact_rollback', xact_rollback,
|
|
'blks_read', blks_read,
|
|
'blks_hit', blks_hit,
|
|
'tup_returned', tup_returned,
|
|
'tup_fetched', tup_fetched,
|
|
'tup_inserted', tup_inserted,
|
|
'tup_updated', tup_updated,
|
|
'tup_deleted', tup_deleted,
|
|
'temp_files', temp_files,
|
|
'temp_bytes', temp_bytes,
|
|
'deadlocks', deadlocks,
|
|
'conflicts', conflicts
|
|
)
|
|
FROM pg_stat_database
|
|
WHERE datname = current_database();
|
|
COMMIT;
|
|
SQL
|
|
}
|
|
|
|
summarize_database_metrics() {
|
|
local before=$1
|
|
local after=$2
|
|
local destination=$3
|
|
|
|
jq -n \
|
|
--slurpfile before "$before" \
|
|
--slurpfile after "$after" '
|
|
def counters:
|
|
[
|
|
"database_size_bytes",
|
|
"xact_commit",
|
|
"xact_rollback",
|
|
"blks_read",
|
|
"blks_hit",
|
|
"tup_returned",
|
|
"tup_fetched",
|
|
"tup_inserted",
|
|
"tup_updated",
|
|
"tup_deleted",
|
|
"temp_files",
|
|
"temp_bytes",
|
|
"deadlocks",
|
|
"conflicts"
|
|
];
|
|
|
|
($before[0]) as $before |
|
|
($after[0]) as $after |
|
|
{
|
|
schema_version: 1,
|
|
measurement: "read-only pg_stat_database snapshots around the k6 run",
|
|
thresholds_applied: false,
|
|
stats_reset_before: $before.stats_reset,
|
|
stats_reset_after: $after.stats_reset,
|
|
before: $before,
|
|
after: $after,
|
|
delta: reduce counters[] as $key
|
|
({}; .[$key] = ($after[$key] - $before[$key]))
|
|
}
|
|
' >"$destination"
|
|
}
|
|
|
|
summarize_resources() {
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
--volume "$ROOT/scripts/summarize-docker-stats.py:/scripts/summarize-docker-stats.py:ro" \
|
|
--volume "$output_dir:/output" \
|
|
"$PYTHON_IMAGE" \
|
|
python /scripts/summarize-docker-stats.py \
|
|
/output/docker-stats.jsonl \
|
|
/output/resource-summary.json
|
|
}
|
|
|
|
sample_database_connections() {
|
|
database_psql >>"$connection_log" <<'SQL'
|
|
WITH activity AS (
|
|
SELECT *
|
|
FROM pg_stat_activity
|
|
WHERE backend_type = 'client backend'
|
|
AND pid <> pg_backend_pid()
|
|
),
|
|
active_wait_events AS (
|
|
SELECT
|
|
concat_ws(':', wait_event_type, wait_event) AS event,
|
|
count(*) AS backends
|
|
FROM activity
|
|
WHERE state = 'active'
|
|
AND wait_event IS NOT NULL
|
|
GROUP BY wait_event_type, wait_event
|
|
)
|
|
SELECT json_build_object(
|
|
'captured_at', clock_timestamp(),
|
|
'max_connections', current_setting('max_connections')::integer,
|
|
'reserved_connections',
|
|
current_setting('reserved_connections')::integer,
|
|
'superuser_reserved_connections',
|
|
current_setting('superuser_reserved_connections')::integer,
|
|
'client_backends',
|
|
count(*),
|
|
'active_backends',
|
|
count(*) FILTER (WHERE state = 'active'),
|
|
'idle_backends',
|
|
count(*) FILTER (WHERE state = 'idle'),
|
|
'idle_in_transaction_backends',
|
|
count(*) FILTER (WHERE state = 'idle in transaction'),
|
|
'active_waiting_backends',
|
|
count(*) FILTER (
|
|
WHERE state = 'active'
|
|
AND wait_event IS NOT NULL
|
|
),
|
|
'lock_waiting_backends',
|
|
count(*) FILTER (WHERE wait_event_type = 'Lock'),
|
|
'active_wait_events',
|
|
(
|
|
SELECT coalesce(json_object_agg(event, backends), '{}'::json)
|
|
FROM active_wait_events
|
|
),
|
|
'longest_idle_in_transaction_ms',
|
|
coalesce(
|
|
max(
|
|
extract(epoch FROM clock_timestamp() - xact_start) * 1000
|
|
) FILTER (
|
|
WHERE state = 'idle in transaction'
|
|
),
|
|
0
|
|
),
|
|
'longest_active_query_ms',
|
|
coalesce(
|
|
max(
|
|
extract(epoch FROM clock_timestamp() - query_start) * 1000
|
|
) FILTER (
|
|
WHERE state = 'active'
|
|
),
|
|
0
|
|
)
|
|
)
|
|
FROM activity;
|
|
SQL
|
|
}
|
|
|
|
summarize_database_connections() {
|
|
jq -s \
|
|
--argjson pool_size "$POOL_SIZE" \
|
|
--argjson app_replicas "$((LOAD_WEB_REPLICAS + LOAD_WORKER_REPLICAS))" '
|
|
. as $samples |
|
|
{
|
|
schema_version: 1,
|
|
measurement: "read-only pg_stat_activity samples during the k6 run",
|
|
thresholds_applied: false,
|
|
sample_count: length,
|
|
configured_app_replicas: $app_replicas,
|
|
configured_pool_size_per_replica: $pool_size,
|
|
configured_repo_pool_slots: ($app_replicas * $pool_size),
|
|
max_connections: (map(.max_connections) | max),
|
|
reserved_connections: (map(.reserved_connections) | max),
|
|
superuser_reserved_connections:
|
|
(map(.superuser_reserved_connections) | max),
|
|
peak_client_backends: (map(.client_backends) | max),
|
|
peak_active_backends: (map(.active_backends) | max),
|
|
peak_idle_backends: (map(.idle_backends) | max),
|
|
peak_idle_in_transaction_backends:
|
|
(map(.idle_in_transaction_backends) | max),
|
|
peak_active_waiting_backends:
|
|
(map(.active_waiting_backends) | max),
|
|
peak_lock_waiting_backends: (map(.lock_waiting_backends) | max),
|
|
observed_active_wait_events:
|
|
([
|
|
$samples[]
|
|
| .active_wait_events
|
|
| to_entries[]
|
|
]
|
|
| group_by(.key)
|
|
| map({
|
|
event: .[0].key,
|
|
peak_backends: (map(.value) | max),
|
|
observed_samples: length
|
|
})),
|
|
longest_observed_idle_in_transaction_ms:
|
|
(map(.longest_idle_in_transaction_ms | tonumber) | max),
|
|
longest_observed_active_query_ms:
|
|
(map(.longest_active_query_ms | tonumber) | max),
|
|
minimum_observed_connection_headroom:
|
|
((map(
|
|
.max_connections -
|
|
.reserved_connections -
|
|
.superuser_reserved_connections
|
|
) | min) -
|
|
(map(.client_backends) | max))
|
|
}
|
|
' "$connection_log" >"$output_dir/database-connections-summary.json"
|
|
}
|
|
|
|
prometheus_value() {
|
|
local metric=$1
|
|
|
|
awk -v metric="$metric" '$1 == metric { print $2; found = 1 }
|
|
END { if (!found) print 0 }'
|
|
}
|
|
|
|
snapshot_web_metrics() {
|
|
local destination=$1
|
|
local temporary
|
|
temporary=$(mktemp "${TMPDIR:-/tmp}/wnh-load-metrics.XXXXXX")
|
|
: >"$temporary"
|
|
|
|
for container in "${web_containers[@]}"; do
|
|
local scrape node queries total execution queue decode
|
|
scrape=$(
|
|
docker exec "$container" sh -c \
|
|
'curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $METRICS_TOKEN" \
|
|
--header "X-Forwarded-Proto: https" \
|
|
--header "Host: $PHX_HOST" \
|
|
http://127.0.0.1:4000/metrics'
|
|
)
|
|
node=$(
|
|
docker exec "$container" /app/bin/who_need_help rpc \
|
|
'IO.puts(JSON.encode!(%{node: to_string(node())}))' |
|
|
tail -n 1 |
|
|
jq -r '.node'
|
|
)
|
|
queries=$(
|
|
printf '%s\n' "$scrape" |
|
|
prometheus_value who_need_help_database_queries_total
|
|
)
|
|
total=$(
|
|
printf '%s\n' "$scrape" |
|
|
prometheus_value who_need_help_database_query_duration_microseconds_total
|
|
)
|
|
execution=$(
|
|
printf '%s\n' "$scrape" |
|
|
prometheus_value \
|
|
who_need_help_database_query_execution_duration_microseconds_total
|
|
)
|
|
queue=$(
|
|
printf '%s\n' "$scrape" |
|
|
prometheus_value \
|
|
who_need_help_database_query_queue_duration_microseconds_total
|
|
)
|
|
decode=$(
|
|
printf '%s\n' "$scrape" |
|
|
prometheus_value \
|
|
who_need_help_database_query_decode_duration_microseconds_total
|
|
)
|
|
|
|
jq -cn \
|
|
--arg container "$container" \
|
|
--arg node "$node" \
|
|
--argjson queries "$queries" \
|
|
--argjson total_duration_us "$total" \
|
|
--argjson execution_duration_us "$execution" \
|
|
--argjson queue_duration_us "$queue" \
|
|
--argjson decode_duration_us "$decode" \
|
|
'{
|
|
container: $container,
|
|
node: $node,
|
|
queries: $queries,
|
|
total_duration_us: $total_duration_us,
|
|
execution_duration_us: $execution_duration_us,
|
|
queue_duration_us: $queue_duration_us,
|
|
decode_duration_us: $decode_duration_us
|
|
}' >>"$temporary"
|
|
done
|
|
|
|
jq -s 'sort_by(.node)' "$temporary" >"$destination"
|
|
rm -f "$temporary"
|
|
}
|
|
|
|
summarize_web_metrics() {
|
|
local before=$1
|
|
local after=$2
|
|
|
|
jq -n \
|
|
--slurpfile before "$before" \
|
|
--slurpfile after "$after" '
|
|
[
|
|
$after[0][] as $after_row
|
|
| ($before[0][] | select(.node == $after_row.node)) as $before_row
|
|
| {
|
|
node: $after_row.node,
|
|
queries: ($after_row.queries - $before_row.queries),
|
|
total_duration_us:
|
|
($after_row.total_duration_us - $before_row.total_duration_us),
|
|
execution_duration_us:
|
|
($after_row.execution_duration_us -
|
|
$before_row.execution_duration_us),
|
|
queue_duration_us:
|
|
($after_row.queue_duration_us - $before_row.queue_duration_us),
|
|
decode_duration_us:
|
|
($after_row.decode_duration_us - $before_row.decode_duration_us)
|
|
}
|
|
| . + {
|
|
average_total_duration_us:
|
|
(if .queries > 0 then .total_duration_us / .queries else 0 end),
|
|
average_execution_duration_us:
|
|
(if .queries > 0 then .execution_duration_us / .queries else 0 end),
|
|
average_queue_duration_us:
|
|
(if .queries > 0 then .queue_duration_us / .queries else 0 end),
|
|
average_decode_duration_us:
|
|
(if .queries > 0 then .decode_duration_us / .queries else 0 end)
|
|
}
|
|
] as $nodes
|
|
| {
|
|
schema_version: 1,
|
|
measurement:
|
|
"per-web Ecto telemetry counter deltas around the k6 run",
|
|
thresholds_applied: false,
|
|
nodes: $nodes,
|
|
cluster: (
|
|
reduce $nodes[] as $node
|
|
({
|
|
queries: 0,
|
|
total_duration_us: 0,
|
|
execution_duration_us: 0,
|
|
queue_duration_us: 0,
|
|
decode_duration_us: 0
|
|
};
|
|
.queries += $node.queries |
|
|
.total_duration_us += $node.total_duration_us |
|
|
.execution_duration_us += $node.execution_duration_us |
|
|
.queue_duration_us += $node.queue_duration_us |
|
|
.decode_duration_us += $node.decode_duration_us)
|
|
| . + {
|
|
average_total_duration_us:
|
|
(if .queries > 0 then .total_duration_us / .queries else 0 end),
|
|
average_execution_duration_us:
|
|
(if .queries > 0
|
|
then .execution_duration_us / .queries else 0 end),
|
|
average_queue_duration_us:
|
|
(if .queries > 0 then .queue_duration_us / .queries else 0 end),
|
|
average_decode_duration_us:
|
|
(if .queries > 0 then .decode_duration_us / .queries else 0 end)
|
|
}
|
|
)
|
|
}
|
|
' >"$output_dir/ecto-query-metrics-summary.json"
|
|
}
|
|
|
|
run_fixture_tool() {
|
|
local action=$1
|
|
|
|
docker run --rm \
|
|
--network "$internal_network_id" \
|
|
--env-file "$ENV_FILE" \
|
|
--env APP_ROLE=migrate \
|
|
--env "WNH_LOAD_EXPECTED_DATABASE=$POSTGRES_DB" \
|
|
--env WNH_LOAD_FIXTURE_CONFIRM=isolated-load-fixtures \
|
|
--env "WNH_LOAD_FIXTURE_RUN_ID=$LABEL" \
|
|
--env "WNH_LOAD_FIXTURE_COUNT=$fixture_count" \
|
|
--env "WNH_LOAD_FIXTURE_PATH=/output/fixtures.json" \
|
|
--volume "$output_dir:/output" \
|
|
who-need-help:load-tools \
|
|
mix wnh.load_fixtures "$action"
|
|
}
|
|
|
|
validate_authenticated_writes() {
|
|
# The run label was restricted to a conservative character set above and is
|
|
# passed as a psql variable, not interpolated into SQL syntax.
|
|
# shellcheck disable=SC2016
|
|
"${compose[@]}" exec -T db sh -c \
|
|
'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
|
--set run_id="$1" --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
|
|
sh "$LABEL" >"$output_dir/authenticated-writes.json" <<'SQL'
|
|
SELECT json_build_object(
|
|
'messages',
|
|
(SELECT count(*) FROM messages WHERE body LIKE 'load:' || :'run_id' || ':%'),
|
|
'tracking_samples',
|
|
(SELECT coalesce(sum(session.sample_count), 0)
|
|
FROM tracking_sessions AS session
|
|
JOIN users AS actor ON actor.id = session.user_id
|
|
WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
|
|
'wnh-load-' || lower(:'run_id') || '-'),
|
|
'tracking_sessions',
|
|
(SELECT count(*)
|
|
FROM tracking_sessions AS session
|
|
JOIN users AS actor ON actor.id = session.user_id
|
|
WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
|
|
'wnh-load-' || lower(:'run_id') || '-'),
|
|
'tracking_positions',
|
|
(SELECT count(*)
|
|
FROM tracking_positions AS position
|
|
JOIN tracking_sessions AS session ON session.id = position.tracking_session_id
|
|
JOIN users AS actor ON actor.id = session.user_id
|
|
WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
|
|
'wnh-load-' || lower(:'run_id') || '-'),
|
|
'session_tokens',
|
|
(SELECT count(*)
|
|
FROM users_tokens AS token
|
|
JOIN users AS actor ON actor.id = token.user_id
|
|
WHERE left(actor.email, length('wnh-load-' || lower(:'run_id') || '-')) =
|
|
'wnh-load-' || lower(:'run_id') || '-')
|
|
);
|
|
SQL
|
|
|
|
jq -e '
|
|
.messages > 0 and
|
|
.tracking_samples > 0 and
|
|
.tracking_sessions > 0 and
|
|
.tracking_positions == 0 and
|
|
.session_tokens > 0
|
|
' "$output_dir/authenticated-writes.json" >/dev/null
|
|
}
|
|
|
|
sample_resources() {
|
|
while [[ -e "$running_marker" ]]; do
|
|
observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
docker stats --no-stream --format '{{json .}}' "${measured_containers[@]}" |
|
|
jq -c --arg observed_at "$observed_at" '. + {ObservedAt: $observed_at}' \
|
|
>>"$resource_log"
|
|
sample_database_connections
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
cleanup_sampler() {
|
|
rm -f "$running_marker"
|
|
|
|
if [[ -n "${sampler_pid:-}" ]]; then
|
|
wait "$sampler_pid" 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
fixtures_prepared=false
|
|
|
|
cleanup_on_exit() {
|
|
local status=$?
|
|
local cleanup_status=0
|
|
|
|
trap - EXIT HUP INT TERM
|
|
cleanup_sampler
|
|
|
|
if [[ "$fixtures_prepared" == true ]]; then
|
|
set +e
|
|
run_fixture_tool cleanup >>"$output_dir/fixture-cleanup.log" 2>&1
|
|
cleanup_status=$?
|
|
set -e
|
|
fi
|
|
|
|
if [[ "$cleanup_status" -ne 0 ]]; then
|
|
echo "Fixture cleanup failed; inspect $output_dir/fixture-cleanup.log." >&2
|
|
status=1
|
|
fi
|
|
|
|
exit "$status"
|
|
}
|
|
|
|
trap cleanup_on_exit EXIT HUP INT TERM
|
|
|
|
{
|
|
printf 'observed_at=%s\n' "$run_started_at"
|
|
printf 'k6_image=%s\n' "$K6_IMAGE"
|
|
printf 'resource_summarizer_image=%s\n' "$PYTHON_IMAGE"
|
|
printf 'load_project=%s\n' "$LOAD_PROJECT"
|
|
printf 'web_replicas=%s\n' "$LOAD_WEB_REPLICAS"
|
|
printf 'worker_replicas=%s\n' "$LOAD_WORKER_REPLICAS"
|
|
printf 'http_vus=%s\n' "$LOAD_HTTP_VUS"
|
|
printf 'websocket_vus=%s\n' "$LOAD_WS_VUS"
|
|
printf 'duration=%s\n' "$LOAD_DURATION"
|
|
printf 'duration_source=%s\n' "$duration_source"
|
|
printf 'websocket_hold_ms=%s\n' "$LOAD_WS_HOLD_MS"
|
|
printf 'websocket_connect_timeout_ms=%s\n' "$LOAD_WS_CONNECT_TIMEOUT_MS"
|
|
printf 'http_think_seconds=%s\n' "$LOAD_HTTP_THINK_SECONDS"
|
|
printf 'authenticated_vus=%s\n' "$LOAD_AUTH_VUS"
|
|
printf 'fixture_count=%s\n' "$fixture_count"
|
|
printf 'authenticated_websocket_timeout_ms=%s\n' "$LOAD_AUTH_WS_TIMEOUT_MS"
|
|
printf 'authenticated_think_seconds=%s\n' "$LOAD_AUTH_THINK_SECONDS"
|
|
docker info --format 'docker_cpus={{.NCPU}} docker_memory_bytes={{.MemTotal}} docker_server={{.ServerVersion}}'
|
|
docker compose version
|
|
uname -a
|
|
} >"$output_dir/environment.txt"
|
|
|
|
prepare_statement_stats
|
|
snapshot_database "$output_dir/database-pre-fixtures.txt"
|
|
run_fixture_tool prepare >"$output_dir/fixture-prepare.log" 2>&1
|
|
fixtures_prepared=true
|
|
snapshot_database "$output_dir/database-before-load.txt"
|
|
snapshot_database_metrics "$output_dir/database-metrics-before-load.json"
|
|
reset_statement_stats
|
|
snapshot_web_metrics "$output_dir/ecto-query-metrics-before.json"
|
|
sample_resources &
|
|
sampler_pid=$!
|
|
|
|
set +e
|
|
docker run --rm \
|
|
--user "$(id -u):$(id -g)" \
|
|
--network "$network_id" \
|
|
--volume "$ROOT/load/k6:/scripts:ro" \
|
|
--volume "$output_dir:/fixtures:ro" \
|
|
--volume "$output_dir:/output" \
|
|
--env "BASE_URL=https://$LOAD_HOST" \
|
|
--env "PUBLIC_ORIGIN=https://$LOAD_HOST" \
|
|
--env "HTTP_VUS=$LOAD_HTTP_VUS" \
|
|
--env "WS_VUS=$LOAD_WS_VUS" \
|
|
--env "DURATION=$LOAD_DURATION" \
|
|
--env "WS_HOLD_MS=$LOAD_WS_HOLD_MS" \
|
|
--env "WS_CONNECT_TIMEOUT_MS=$LOAD_WS_CONNECT_TIMEOUT_MS" \
|
|
--env "HTTP_THINK_SECONDS=$LOAD_HTTP_THINK_SECONDS" \
|
|
--env "AUTH_VUS=$LOAD_AUTH_VUS" \
|
|
--env "AUTH_WS_TIMEOUT_MS=$LOAD_AUTH_WS_TIMEOUT_MS" \
|
|
--env "AUTH_THINK_SECONDS=$LOAD_AUTH_THINK_SECONDS" \
|
|
--env "FIXTURE_PASSWORD=$LOAD_FIXTURE_PASSWORD" \
|
|
--env "FIXTURE_RUN_ID=$LABEL" \
|
|
"$K6_IMAGE" run \
|
|
--no-usage-report \
|
|
--summary-mode=full \
|
|
--summary-export=/output/k6-summary.json \
|
|
--new-machine-readable-summary \
|
|
/scripts/public-and-websocket.js 2>&1 |
|
|
tee "$output_dir/k6.log"
|
|
k6_status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
cleanup_sampler
|
|
sampler_pid=
|
|
summarize_resources
|
|
summarize_database_connections
|
|
snapshot_web_metrics "$output_dir/ecto-query-metrics-after.json"
|
|
summarize_web_metrics \
|
|
"$output_dir/ecto-query-metrics-before.json" \
|
|
"$output_dir/ecto-query-metrics-after.json"
|
|
snapshot_statement_stats "$output_dir/statement-statistics.json"
|
|
|
|
snapshot_database "$output_dir/database-after-load.txt"
|
|
snapshot_database_metrics "$output_dir/database-metrics-after-load.json"
|
|
summarize_database_metrics \
|
|
"$output_dir/database-metrics-before-load.json" \
|
|
"$output_dir/database-metrics-after-load.json" \
|
|
"$output_dir/database-metrics-summary.json"
|
|
diff -u "$output_dir/database-before-load.txt" "$output_dir/database-after-load.txt" \
|
|
>"$output_dir/database-load-diff.txt" || true
|
|
validate_authenticated_writes
|
|
|
|
COMPOSE_PROJECT_NAME=$LOAD_PROJECT "$ROOT/scripts/verify-realtime-cluster.sh" compose \
|
|
>"$output_dir/pubsub-probe.txt"
|
|
curl --fail --silent --show-error \
|
|
--header "Host: $LOAD_HOST" \
|
|
"http://localhost:$HTTP_PORT/healthz/ready" \
|
|
>"$output_dir/readiness-after.json"
|
|
"${compose[@]}" ps -a >"$output_dir/compose-after.txt"
|
|
"${compose[@]}" logs --since "$run_started_at" proxy web worker \
|
|
>"$output_dir/application.log" 2>&1
|
|
|
|
if ! jq -e '
|
|
def metric($name):
|
|
([.results.metrics[] | select(.name == $name) | .values][0] // {});
|
|
|
|
(.results.checks.metrics[] |
|
|
select(.name == "checks_failed") |
|
|
.values.matches) == 0 and
|
|
metric("http_req_failed").matches == 0 and
|
|
metric("wnh_websocket_opened").count > 0 and
|
|
metric("wnh_websocket_opened").count ==
|
|
metric("wnh_websocket_heartbeat_replies").count and
|
|
(metric("wnh_websocket_errors").count // 0) == 0 and
|
|
metric("wnh_authenticated_logins").count > 0 and
|
|
metric("wnh_authenticated_pages").count > 0 and
|
|
metric("wnh_tracking_updates").count > 0 and
|
|
metric("wnh_liveview_joins").count > 0 and
|
|
metric("wnh_liveview_joins").count ==
|
|
metric("wnh_liveview_tracking_starts").count and
|
|
metric("wnh_liveview_joins").count == metric("wnh_tracking_updates").count and
|
|
metric("wnh_liveview_joins").count == metric("wnh_liveview_messages").count and
|
|
metric("wnh_liveview_joins").count ==
|
|
metric("wnh_liveview_tracking_stops").count and
|
|
(metric("wnh_authenticated_errors").count // 0) == 0
|
|
' "$output_dir/k6-summary.json" >/dev/null; then
|
|
echo "Functional load checks failed; evidence is in $output_dir." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$k6_status" -ne 0 ]]; then
|
|
echo "k6 exited with status $k6_status; evidence is in $output_dir." >&2
|
|
exit "$k6_status"
|
|
fi
|
|
|
|
run_fixture_tool cleanup >"$output_dir/fixture-cleanup.log" 2>&1
|
|
fixtures_prepared=false
|
|
snapshot_database "$output_dir/database-after-cleanup.txt"
|
|
snapshot_database_metrics "$output_dir/database-metrics-after-cleanup.json"
|
|
|
|
if ! diff -u "$output_dir/database-pre-fixtures.txt" \
|
|
"$output_dir/database-after-cleanup.txt" >"$output_dir/database-cleanup-diff.txt"; then
|
|
echo "Fixture cleanup did not restore the pre-run table counts; evidence is in $output_dir." >&2
|
|
exit 1
|
|
fi
|
|
|
|
trap - EXIT HUP INT TERM
|
|
printf 'Load evidence: %s\n' "$output_dir"
|