who_need_help/scripts/production-readonly-load.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
K6_IMAGE="grafana/k6@sha256:65c920dc067d5e2e00befbf982af6ad6ad0117034e8b1c65817c7975c52d4669"
BASE_URL="https://whoneedhelp.com"
SSH_TARGET=${WNH_PRODUCTION_LOAD_SSH_TARGET:-whoneedhelp}
REMOTE_DIRECTORY=${WNH_PRODUCTION_LOAD_REMOTE_DIRECTORY:-/srv/who_need_help-production}
MODE=${1:-plan}
LABEL=${2:-"production-readonly-$(date -u +%Y%m%dT%H%M%SZ)"}

usage() {
  cat <<'EOF'
Usage:
  WNH_PRODUCTION_LOAD_HTTP_VUS=... \
  WNH_PRODUCTION_LOAD_WS_VUS=... \
  WNH_PRODUCTION_LOAD_DURATION=... \
  WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS=... \
  WNH_PRODUCTION_LOAD_WS_HOLD_MS=... \
  WNH_PRODUCTION_LOAD_WS_CONNECT_TIMEOUT_MS=... \
    ./scripts/production-readonly-load.sh plan [label]

The run mode additionally requires the exact confirmation string printed by
plan in WNH_PRODUCTION_LOAD_CONFIRM.
EOF
}

case "$MODE" in
  plan | run) ;;
  *)
    usage >&2
    exit 1
    ;;
esac

if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
  echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
  exit 1
fi

required=(
  WNH_PRODUCTION_LOAD_HTTP_VUS
  WNH_PRODUCTION_LOAD_WS_VUS
  WNH_PRODUCTION_LOAD_DURATION
  WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS
  WNH_PRODUCTION_LOAD_WS_HOLD_MS
  WNH_PRODUCTION_LOAD_WS_CONNECT_TIMEOUT_MS
)

for name in "${required[@]}"; do
  if [[ -z "${!name:-}" ]]; then
    echo "$name must be supplied explicitly for this experiment." >&2
    exit 1
  fi
done

for name in WNH_PRODUCTION_LOAD_HTTP_VUS WNH_PRODUCTION_LOAD_WS_VUS; do
  if [[ ! "${!name}" =~ ^[0-9]+$ ]]; then
    echo "$name must be a non-negative integer." >&2
    exit 1
  fi
done

if ((WNH_PRODUCTION_LOAD_HTTP_VUS + WNH_PRODUCTION_LOAD_WS_VUS == 0)); then
  echo "At least one HTTP or WebSocket VU is required." >&2
  exit 1
fi

if [[ ! "$WNH_PRODUCTION_LOAD_DURATION" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then
  echo "WNH_PRODUCTION_LOAD_DURATION must be a positive k6 duration." >&2
  exit 1
fi

if [[ ! "$WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
  echo "WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS must be non-negative." >&2
  exit 1
fi

for name in \
  WNH_PRODUCTION_LOAD_WS_HOLD_MS \
  WNH_PRODUCTION_LOAD_WS_CONNECT_TIMEOUT_MS; do
  if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then
    echo "$name must be a positive integer." >&2
    exit 1
  fi
done

inventory=$(
  ssh "$SSH_TARGET" bash -s -- "$REMOTE_DIRECTORY" <<'REMOTE'
set -euo pipefail
directory=$1
env_file="$directory/.env"

if [[ ! -f "$env_file" ]]; then
  echo "Missing production environment: $env_file" >&2
  exit 1
fi

read_env() {
  local name=$1
  sed -n "s/^${name}=//p" "$env_file" | tail -n 1
}

deployment_env=$(read_env DEPLOYMENT_ENV)
phx_host=$(read_env PHX_HOST)
base_url=$(read_env WNH_BASE_URL)
project=$(read_env COMPOSE_PROJECT_NAME)

if [[ "$deployment_env" != "production" ]] ||
  [[ "$phx_host" != "whoneedhelp.com" ]] ||
  [[ "$base_url" != "https://whoneedhelp.com" ]] ||
  [[ "$project" != "who_need_help_production" ]]; then
  echo "The remote checkout did not match the pinned production identity." >&2
  exit 1
fi

commit=$(git -C "$directory" rev-parse HEAD)
mapfile -t containers < <(
  docker ps \
    --filter "label=com.docker.compose.project=$project" \
    --filter "label=com.docker.compose.service=app" \
    --format '{{.Names}}'
)

if [[ "${#containers[@]}" -eq 0 ]]; then
  mapfile -t containers < <(
    docker ps \
      --filter "label=com.docker.compose.project=$project" \
      --filter "label=com.docker.compose.service=web" \
      --format '{{.Names}}'
  )
fi

if [[ "${#containers[@]}" -eq 0 ]]; then
  echo "No running production application containers were found." >&2
  exit 1
fi

printf 'commit=%s\n' "$commit"
printf 'project=%s\n' "$project"
printf 'deployment_env=%s\n' "$deployment_env"
printf 'phx_host=%s\n' "$phx_host"
printf 'base_url=%s\n' "$base_url"
printf 'containers=%s\n' "$(IFS=,; echo "${containers[*]}")"
printf 'cpu_count=%s\n' "$(getconf _NPROCESSORS_ONLN)"
printf 'mem_total_kib=%s\n' "$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)"
printf 'mem_available_kib=%s\n' "$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)"
printf 'load_average=%s\n' "$(cut -d' ' -f1-3 /proc/loadavg)"
REMOTE
)

value_from_inventory() {
  local name=$1
  printf '%s\n' "$inventory" | sed -n "s/^${name}=//p" | tail -n 1
}

commit=$(value_from_inventory commit)
project=$(value_from_inventory project)
containers_csv=$(value_from_inventory containers)
deployment_env=$(value_from_inventory deployment_env)
phx_host=$(value_from_inventory phx_host)
verified_base_url=$(value_from_inventory base_url)

if [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]]; then
  echo "Could not verify the production commit." >&2
  exit 1
fi
if [[ "$project" != "who_need_help_production" ]] ||
  [[ "$deployment_env" != "production" ]] ||
  [[ "$phx_host" != "whoneedhelp.com" ]] ||
  [[ "$verified_base_url" != "$BASE_URL" ]] ||
  [[ -z "$containers_csv" ]]; then
  echo "The verified inventory does not match the pinned production identity." >&2
  exit 1
fi

confirmation="whoneedhelp.com:${commit}:http=${WNH_PRODUCTION_LOAD_HTTP_VUS}:ws=${WNH_PRODUCTION_LOAD_WS_VUS}:duration=${WNH_PRODUCTION_LOAD_DURATION}"

cat <<EOF
Verified target:
  SSH target: $SSH_TARGET
  directory: $REMOTE_DIRECTORY
  URL: $BASE_URL
  Compose project: $project
  commit: $commit
  application containers: $containers_csv
  CPU count: $(value_from_inventory cpu_count)
  MemTotal: $(value_from_inventory mem_total_kib) KiB
  MemAvailable now: $(value_from_inventory mem_available_kib) KiB
  load average now: $(value_from_inventory load_average)

Exact request scope:
  GET /, /safety, /privacy, /terms, /users/log-in,
      /users/register, /healthz/ready
  WebSocket /live/websocket with Phoenix heartbeat only
  No POST, login attempt, registration, email, support request, chat,
  location update, application-data fixture, migration, or database reset.

Experiment inputs:
  HTTP VUs: $WNH_PRODUCTION_LOAD_HTTP_VUS
  WebSocket VUs: $WNH_PRODUCTION_LOAD_WS_VUS
  duration: $WNH_PRODUCTION_LOAD_DURATION
  HTTP think seconds: $WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS
  WebSocket hold ms: $WNH_PRODUCTION_LOAD_WS_HOLD_MS
  WebSocket connect timeout ms: $WNH_PRODUCTION_LOAD_WS_CONNECT_TIMEOUT_MS

To execute exactly this plan, set:
  WNH_PRODUCTION_LOAD_CONFIRM='$confirmation'
EOF

if [[ "$MODE" == "plan" ]]; then
  exit 0
fi

if [[ "${WNH_PRODUCTION_LOAD_CONFIRM:-}" != "$confirmation" ]]; then
  echo "Run refused: WNH_PRODUCTION_LOAD_CONFIRM does not match this verified plan." >&2
  exit 1
fi

output_dir="$ROOT/output/performance/$LABEL"
if [[ -e "$output_dir" ]]; then
  echo "Output path already exists: $output_dir" >&2
  exit 1
fi
mkdir -p "$output_dir"
chmod 700 "$ROOT/output" "$ROOT/output/performance" "$output_dir"

printf '%s\n' "$inventory" >"$output_dir/environment.txt"
cat >"$output_dir/scope.txt" <<EOF
target=$BASE_URL
commit=$commit
confirmation=$confirmation
http_vus=$WNH_PRODUCTION_LOAD_HTTP_VUS
websocket_vus=$WNH_PRODUCTION_LOAD_WS_VUS
duration=$WNH_PRODUCTION_LOAD_DURATION
http_think_seconds=$WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS
websocket_hold_ms=$WNH_PRODUCTION_LOAD_WS_HOLD_MS
websocket_connect_timeout_ms=$WNH_PRODUCTION_LOAD_WS_CONNECT_TIMEOUT_MS
scope=public GET allowlist and Phoenix heartbeat only
latency_or_capacity_thresholds_applied=false
EOF

IFS=, read -r -a containers <<<"$containers_csv"
primary_container=${containers[0]}

snapshot_database() {
  local destination=$1
  ssh "$SSH_TARGET" bash -s -- "$primary_container" <<'REMOTE' | tail -n 1 >"$destination"
set -euo pipefail
container=$1
docker exec "$container" /app/bin/who_need_help rpc '
      alias WhoNeedHelp.Repo
      database = Repo.query!("""
      SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit,
             temp_files, temp_bytes, deadlocks
      FROM pg_stat_database
      WHERE datname = current_database()
      """).rows
      counts = Repo.query!("""
      SELECT
        (SELECT count(*) FROM users),
        (SELECT count(*) FROM users_tokens),
        (SELECT count(*) FROM rate_limit_buckets)
      """).rows
      IO.puts(Jason.encode!(%{
        captured_at: DateTime.utc_now(),
        pg_stat_database: database,
        counts: counts,
        tables: ["users", "users_tokens", "rate_limit_buckets"]
      }))
    '
REMOTE
}

snapshot_metrics() {
  local destination=$1
  # Expansion of METRICS_TOKEN occurs only inside the application container.
  # shellcheck disable=SC2016
  ssh "$SSH_TARGET" bash -s -- "$primary_container" <<'REMOTE' >"$destination"
set -euo pipefail
container=$1
docker exec "$container" sh -c \
    'curl --fail --silent --show-error \
      --header "Authorization: Bearer $METRICS_TOKEN" \
      http://127.0.0.1:4000/metrics'
REMOTE
}

resource_log="$output_dir/server-stats.jsonl"
sampler_pid=
cleanup() {
  local status=$?
  trap - EXIT HUP INT TERM
  if [[ -n "$sampler_pid" ]] && kill -0 "$sampler_pid" 2>/dev/null; then
    kill "$sampler_pid" 2>/dev/null || true
    wait "$sampler_pid" 2>/dev/null || true
  fi
  exit "$status"
}
trap cleanup EXIT HUP INT TERM

curl --fail --silent --show-error "$BASE_URL/healthz/ready" \
  >"$output_dir/readiness-before.json"
snapshot_database "$output_dir/database-before.json"
snapshot_metrics "$output_dir/metrics-before.prom"

ssh "$SSH_TARGET" bash -s -- "$project" <<'REMOTE' >"$resource_log" &
set -euo pipefail
project=$1
while :; do
  observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  load_1=$(awk '{print $1}' /proc/loadavg)
  mem_total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
  mem_available=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
  host_postgres_rss=$(
    ps -C postgres -o rss= 2>/dev/null |
      awk '{sum += $1} END {print sum + 0}'
  )
  mapfile -t ids < <(
    docker ps -q --filter "label=com.docker.compose.project=$project"
  )
  if [[ "${#ids[@]}" -eq 0 ]]; then
    exit 1
  fi
  containers=$(
    docker stats --no-stream --format '{{json .}}' "${ids[@]}" |
      jq -s .
  )
  jq -cn \
    --arg observed_at "$observed_at" \
    --arg load_1 "$load_1" \
    --arg mem_total "$mem_total" \
    --arg mem_available "$mem_available" \
    --arg host_postgres_rss "$host_postgres_rss" \
    --argjson containers "$containers" \
    '{
      observed_at: $observed_at,
      host: {
        load_1: ($load_1 | tonumber),
        mem_total_kib: ($mem_total | tonumber),
        mem_available_kib: ($mem_available | tonumber),
        host_postgres_rss_kib: ($host_postgres_rss | tonumber)
      },
      containers: $containers
    }'
  sleep 1
done
REMOTE
sampler_pid=$!

set +e
docker run --rm \
  --user "$(id -u):$(id -g)" \
  --volume "$ROOT/load/k6:/scripts:ro" \
  --volume "$output_dir:/results" \
  --env BASE_URL="$BASE_URL" \
  --env HTTP_VUS="$WNH_PRODUCTION_LOAD_HTTP_VUS" \
  --env WS_VUS="$WNH_PRODUCTION_LOAD_WS_VUS" \
  --env DURATION="$WNH_PRODUCTION_LOAD_DURATION" \
  --env HTTP_THINK_SECONDS="$WNH_PRODUCTION_LOAD_HTTP_THINK_SECONDS" \
  --env WS_HOLD_MS="$WNH_PRODUCTION_LOAD_WS_HOLD_MS" \
  --env WS_CONNECT_TIMEOUT_MS="$WNH_PRODUCTION_LOAD_WS_CONNECT_TIMEOUT_MS" \
  --env K6_NO_USAGE_REPORT=true \
  "$K6_IMAGE" run \
  --no-usage-report \
  --summary-mode=full \
  --summary-export /results/k6-summary.json \
  --new-machine-readable-summary \
  /scripts/production-readonly.js |
  tee "$output_dir/k6-console.txt"
k6_status=${PIPESTATUS[0]}
set -e

kill "$sampler_pid" 2>/dev/null || true
wait "$sampler_pid" 2>/dev/null || true
sampler_pid=

curl --fail --silent --show-error "$BASE_URL/healthz/ready" \
  >"$output_dir/readiness-after.json"
snapshot_database "$output_dir/database-after.json"
snapshot_metrics "$output_dir/metrics-after.prom"
"$ROOT/scripts/summarize-production-load.py" \
  "$resource_log" "$output_dir/server-stats-summary.json"

jq -e \
  --argjson http_vus "$WNH_PRODUCTION_LOAD_HTTP_VUS" \
  --argjson websocket_vus "$WNH_PRODUCTION_LOAD_WS_VUS" '
  def metric($name):
    ([.results.metrics[] | select(.name == $name) | .values][0] // {});

  (
    $http_vus == 0 or
    (
      (.results.checks.metrics[] |
        select(.name == "checks_failed") |
        .values.matches) == 0 and
      metric("http_req_failed").matches == 0
    )
  ) and
  (
    $websocket_vus == 0 or
    (
      (metric("wnh_readonly_websocket_errors").count // 0) == 0 and
      metric("wnh_readonly_websocket_opened").count > 0 and
      metric("wnh_readonly_websocket_opened").count ==
        metric("wnh_readonly_heartbeat_replies").count
    )
  )
' "$output_dir/k6-summary.json" >/dev/null

if [[ "$k6_status" -ne 0 ]]; then
  echo "k6 exited with status $k6_status; evidence is in $output_dir." >&2
  exit "$k6_status"
fi

echo "Production read-only probe completed. Evidence: $output_dir"