#!/usr/bin/env bash set -euo pipefail umask 077 # Production browser verification is intentionally opt-in and run-scoped. ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) MODE=${1:-plan} RUN_ID=${2:-"production-e2e-$(date -u +%Y%m%d%H%M%S)"} SSH_TARGET=${WNH_PRODUCTION_E2E_SSH_TARGET:-whoneedhelp} REMOTE_ROOT=${WNH_PRODUCTION_E2E_REMOTE_ROOT:-/srv/who_need_help-production} BASE_URL=https://whoneedhelp.com usage() { cat <<'EOF' Usage: ./scripts/production-full-e2e.sh plan [run-id] WNH_PRODUCTION_E2E_CONFIRM='' \ ./scripts/production-full-e2e.sh run [run-id] The run creates only run-scoped synthetic users and records, exercises the two-user help and moderated activity browser flows, and removes the exact fixture on success, failure, or interrupt. It never resets the database. EOF } case "$MODE" in plan | run) ;; *) usage >&2; exit 1 ;; esac if [[ ! "$RUN_ID" =~ ^[a-z0-9-]+$ ]]; then echo "run-id may contain only lowercase letters, numbers, and dashes." >&2 exit 1 fi for command in curl docker git jq mktemp openssl scp sha256sum ssh tar; do command -v "$command" >/dev/null 2>&1 || { echo "Required command is unavailable: $command" >&2 exit 2 } done inventory=$( ssh -o BatchMode=yes "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$RUN_ID" <<'REMOTE' set -euo pipefail root=$1 run_id=$2 env_file="$root/.env" if [[ ! -f "$env_file" ]]; then echo "Missing production environment: $env_file" >&2 exit 1 fi python3 - "$env_file" <<'PY' import shlex import sys path = sys.argv[1] wanted = { "COMPOSE_PROJECT_NAME", "DATABASE_MODE", "DEPLOYMENT_ENV", "PHX_HOST", "POSTGRES_DB", "WNH_BASE_URL", } values = {} with open(path, encoding="utf-8") as handle: for raw_line in handle: line = raw_line.strip() if not line or line.startswith("#") or "=" not in line: continue key, value = line.split("=", 1) if key not in wanted: continue parsed = shlex.split(value, comments=False, posix=True) values[key] = parsed[0] if parsed else "" missing = sorted(key for key in wanted if not values.get(key)) if missing: raise SystemExit("Missing production identity settings: " + ", ".join(missing)) for key in sorted(wanted): print(f"{key.lower()}={values[key]}") PY commit=$(git -C "$root" rev-parse HEAD) project=$( python3 - "$env_file" <<'PY' import shlex import sys with open(sys.argv[1], encoding="utf-8") as handle: for raw_line in handle: line = raw_line.strip() if line.startswith("COMPOSE_PROJECT_NAME="): parsed = shlex.split(line.split("=", 1)[1], comments=False, posix=True) if parsed: print(parsed[0]) break PY ) if [[ -z "$project" ]]; then echo "Missing normalized COMPOSE_PROJECT_NAME." >&2 exit 1 fi mapfile -t containers < <( docker ps \ --filter "label=com.docker.compose.project=$project" \ --filter "label=com.docker.compose.service=app" \ --format '{{.Names}}' ) if [[ "${#containers[@]}" -ne 1 ]]; then echo "Expected exactly one compact production app container; observed ${#containers[@]}." >&2 exit 1 fi container=${containers[0]} health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") restart_count=$(docker inspect --format '{{.RestartCount}}' "$container") prefix="wnh-staging-e2e-$run_id-%" prefix_count=$( docker exec "$container" /app/bin/who_need_help rpc \ "result = WhoNeedHelp.Repo.query!(\"SELECT count(*) FROM users WHERE email LIKE \\\$1\", [\"$prefix\"], log: false); IO.puts(result.rows |> hd() |> hd())" | tail -n 1 ) printf 'commit=%s\n' "$commit" printf 'container=%s\n' "$container" printf 'health=%s\n' "$health" printf 'restart_count=%s\n' "$restart_count" printf 'fixture_prefix_count=%s\n' "$prefix_count" REMOTE ) value() { local name=$1 printf '%s\n' "$inventory" | sed -n "s/^${name}=//p" | tail -n 1 } commit=$(value commit) container=$(value container) database=$(value postgres_db) project=$(value compose_project_name) if [[ "$(value deployment_env)" != production ]] || [[ "$(value phx_host)" != whoneedhelp.com ]] || [[ "$(value wnh_base_url)" != "$BASE_URL" ]] || [[ "$(value database_mode)" != external ]] || [[ "$project" != who_need_help_production ]] || [[ "$(value health)" != healthy ]] || [[ "$(value fixture_prefix_count)" != 0 ]] || [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]] || [[ -z "$database" ]] || [[ -z "$container" ]]; then echo "The observed target does not match the exact compact production identity." >&2 printf '%s\n' "$inventory" >&2 exit 1 fi git cat-file -e "$commit^{commit}" 2>/dev/null || { echo "Production commit $commit is not available in the local repository." >&2 exit 1 } # The deployed application is always archived from its exact remote commit. # Local development may legitimately be ahead of production; those files never # enter either runner image. Only these explicit verifier overlays are copied # below, and their hashes are retained with the evidence bundle. for verifier_path in \ lib/mix/tasks/wnh.staging_full_e2e.ex \ e2e/tests/activity-moderation.spec.ts \ e2e/tests/helpers.ts; do if [[ ! -f "$ROOT/$verifier_path" ]]; then echo "Production E2E verifier is unavailable: $verifier_path" >&2 exit 1 fi done confirmation="whoneedhelp.com:${commit}:${database}:${RUN_ID}:full-browser-e2e" cat <&2 exit 1 fi output_dir="$ROOT/output/production-full-e2e/$RUN_ID" remote_output="$REMOTE_ROOT/output/production-full-e2e/$RUN_ID" short=${commit:0:12} tools_image="who-need-help:production-e2e-tools-$short" browser_image="who-need-help-e2e-tests:production-$short" archive_dir=$(mktemp -d "$ROOT/tmp/production-e2e-source.XXXXXX") fixture_password=$(openssl rand -base64 36 | tr -d '\n') prepared=0 if [[ -e "$output_dir" ]]; then echo "Local evidence directory already exists: $output_dir" >&2 exit 1 fi mkdir -p "$output_dir/browser" chmod 700 "$ROOT/output" "$ROOT/output/production-full-e2e" "$output_dir" "$output_dir/browser" printf '%s\n' "$inventory" >"$output_dir/environment.txt" printf '%s\n' "$confirmation" >"$output_dir/confirmation.txt" snapshot() { local destination=$1 ssh -o BatchMode=yes "$SSH_TARGET" bash -s -- "$container" "$RUN_ID" <<'REMOTE' >"$destination" set -euo pipefail container=$1 run_id=$2 docker exec "$container" /app/bin/who_need_help rpc ' alias WhoNeedHelp.Repo prefix = "wnh-staging-e2e-'"$run_id"'-%" tables = ~w(users help_requests help_assignments messages tracking_sessions tracking_positions reviews activities activity_participants activity_messages reports category_proposals category_votes audit_events notifications oban_jobs) counts = Map.new(tables, fn table -> result = Repo.query!("SELECT count(*) FROM #{table}", [], log: false) {table, result.rows |> hd() |> hd()} end) prefix_count = Repo.query!("SELECT count(*) FROM users WHERE email LIKE $1", [prefix], log: false).rows |> hd() |> hd() IO.puts(Jason.encode!(%{captured_at: DateTime.utc_now(), counts: counts, fixture_prefix_count: prefix_count})) ' | tail -n 1 REMOTE } run_fixture() { local action=$1 ssh -o BatchMode=yes "$SSH_TARGET" bash -s -- \ "$REMOTE_ROOT" "$remote_output" "$tools_image" "$database" "$RUN_ID" "$fixture_password" "$action" <<'REMOTE' set -euo pipefail root=$1 output=$2 image=$3 database=$4 run_id=$5 password=$6 action=$7 mkdir -p "$output" chmod 700 "$output" docker run --rm \ --network host \ --env-file "$root/.env" \ --env APP_ROLE=migrate \ --env "WNH_STAGING_E2E_EXPECTED_DATABASE=$database" \ --env WNH_STAGING_E2E_CONFIRM=public-staging-full-e2e \ --env "WNH_STAGING_E2E_RUN_ID=$run_id" \ --env "WNH_STAGING_E2E_PASSWORD=$password" \ --env WNH_STAGING_E2E_MANIFEST_PATH=/output/fixture.json \ --volume /var/run/postgresql:/var/run/postgresql \ --volume "$output:/output" \ "$image" \ mix wnh.staging_full_e2e "$action" REMOTE } cleanup() { local status=$? trap - EXIT HUP INT TERM if [[ "$prepared" -eq 1 ]]; then if ! run_fixture cleanup >"$output_dir/fixture-cleanup.log" 2>&1; then echo "Exact production E2E fixture cleanup failed; inspect $output_dir." >&2 status=1 fi fi snapshot "$output_dir/database-after.json" 2>"$output_dir/database-after-error.log" || status=1 if [[ -s "$output_dir/database-after.json" ]] && [[ "$(jq -r '.fixture_prefix_count' "$output_dir/database-after.json")" != 0 ]]; then echo "Run-scoped production fixture users remain after cleanup." >&2 status=1 fi scp -q "$SSH_TARGET:$remote_output/fixture.json" "$output_dir/fixture.json" 2>/dev/null || true ssh -o BatchMode=yes "$SSH_TARGET" \ "rm -rf -- '$remote_output'; docker image rm '$tools_image' >/dev/null 2>&1 || true" || status=1 docker image rm "$tools_image" "$browser_image" >/dev/null 2>&1 || true rm -rf -- "$archive_dir" unset fixture_password exit "$status" } trap cleanup EXIT HUP INT TERM curl --fail --silent --show-error --max-time 10 "$BASE_URL/healthz/ready" \ >"$output_dir/readiness-before.json" snapshot "$output_dir/database-before.json" git archive "$commit" | tar -x -C "$archive_dir" # The deployed application remains the exact production commit. Only the # run-scoped fixture task and browser assertion are overlaid into throwaway # runner images so the verifier can evolve without deploying application code. cp "$ROOT/lib/mix/tasks/wnh.staging_full_e2e.ex" \ "$archive_dir/lib/mix/tasks/wnh.staging_full_e2e.ex" cp "$ROOT/e2e/tests/activity-moderation.spec.ts" \ "$archive_dir/e2e/tests/activity-moderation.spec.ts" cp "$ROOT/e2e/tests/helpers.ts" "$archive_dir/e2e/tests/helpers.ts" sha256sum \ "$ROOT/lib/mix/tasks/wnh.staging_full_e2e.ex" \ "$ROOT/e2e/tests/activity-moderation.spec.ts" \ "$ROOT/e2e/tests/helpers.ts" \ >"$output_dir/harness-sha256.txt" # This script starts with umask 077 so evidence and credentials remain private. # The archived source is copied into a browser image that later runs under the # invoking host UID; make only this throwaway source tree readable first. chmod -R u+rwX,go+rX "$archive_dir" docker build --target load_tools --tag "$tools_image" "$archive_dir" \ >"$output_dir/tools-build.log" docker build --tag "$browser_image" "$archive_dir/e2e" \ >"$output_dir/browser-build.log" docker save "$tools_image" | ssh -o BatchMode=yes "$SSH_TARGET" docker load \ >"$output_dir/tools-load.log" run_fixture prepare >"$output_dir/fixture-prepare.log" prepared=1 docker run --rm \ --network host \ --user "$(id -u):$(id -g)" \ --env "BASE_URL=$BASE_URL" \ --env MAILPIT_URL=http://127.0.0.1:1 \ --env "E2E_RUN_ID=$RUN_ID" \ --env "E2E_FIXTURE_PASSWORD=$fixture_password" \ --env "E2E_ADMIN_EMAIL=wnh-staging-e2e-$RUN_ID-admin@example.invalid" \ --env HOME=/tmp \ --volume "$output_dir/browser:/work/output" \ "$browser_image" \ npx playwright test --project=chromium \ tests/mutual-aid.spec.ts tests/activity-moderation.spec.ts | tee "$output_dir/browser-console.log" curl --fail --silent --show-error --max-time 10 "$BASE_URL/healthz/ready" \ >"$output_dir/readiness-after-browser.json" echo "Production full browser E2E passed. Evidence: $output_dir"