#!/bin/sh set -eu umask 077 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) cd "$ROOT" if [ "$#" -ne 1 ]; then echo "Usage: $0 PATH_TO_CUSTOM_FORMAT_DUMP" >&2 exit 1 fi dump=$1 checksum="$dump.sha256" if [ ! -f "$dump" ] || [ ! -s "$dump" ]; then echo "Dump does not exist or is empty: $dump" >&2 exit 1 fi if [ ! -f "$checksum" ]; then echo "Checksum manifest is required: $checksum" >&2 exit 1 fi "$ROOT/scripts/ensure-local-e2e-env.sh" >/dev/null PUBLIC_ENV_FILE="$ROOT/.env" ENV_FILE="$ROOT/.env.e2e" if [ ! -f "$PUBLIC_ENV_FILE" ]; then echo "Missing $PUBLIC_ENV_FILE." >&2 exit 1 fi rehearsal_phx_host=$( sh -c '. "$1"; printf %s "${PHX_HOST:?PHX_HOST is missing from .env}"' \ sh "$PUBLIC_ENV_FILE" ) rehearsal_phx_scheme=$( sh -c '. "$1"; printf %s "${PHX_SCHEME:?PHX_SCHEME is missing from .env}"' \ sh "$PUBLIC_ENV_FILE" ) rehearsal_phx_url_port=$( sh -c '. "$1"; printf %s "${PHX_URL_PORT:?PHX_URL_PORT is missing from .env}"' \ sh "$PUBLIC_ENV_FILE" ) rehearsal_map_tile_url=$( sh -c \ '. "$1"; printf %s "${MAP_TILE_URL:-https://tile.openstreetmap.org/{z}/{x}/{y}.png}"' \ sh "$PUBLIC_ENV_FILE" ) set -a # shellcheck source=/dev/null . "$ENV_FILE" set +a : "${POSTGRES_USER:?POSTGRES_USER is missing from .env.e2e}" : "${DATABASE_URL:?DATABASE_URL is missing from .env.e2e}" dump_dir=$(CDPATH='' cd -- "$(dirname -- "$dump")" && pwd) dump_name=$(basename -- "$dump") ( cd "$dump_dir" sha256sum --check --status "$(basename -- "$checksum")" ) run_id="$(date -u +%Y%m%d%H%M%S)-$$" compact_id=$(printf '%s' "$run_id" | tr -d '-') project="who_need_help_upgrade_rehearsal_$compact_id" rehearsal_db="wnh_upgrade_rehearsal_$compact_id" output_dir="$ROOT/output/upgrade-rehearsal/$run_id" case "$project:$rehearsal_db" in who_need_help_upgrade_rehearsal_*:wnh_upgrade_rehearsal_*) ;; *) echo "Refusing unexpected rehearsal identifiers." >&2 exit 1 ;; esac mkdir -p "$output_dir" chmod 700 "$ROOT/output" "$ROOT/output/upgrade-rehearsal" "$output_dir" case "$DATABASE_URL" in *\?*) database_base=${DATABASE_URL%%\?*} database_query="?${DATABASE_URL#*\?}" ;; *) database_base=$DATABASE_URL database_query= ;; esac database_prefix=${database_base%/*} if [ "$database_prefix" = "$database_base" ]; then echo "Could not derive the isolated rehearsal DATABASE_URL." >&2 exit 1 fi export DATABASE_URL="$database_prefix/$rehearsal_db$database_query" export HTTP_PORT=0 export MAILPIT_PORT=0 export DOCKER_SOCKET_GID DOCKER_SOCKET_GID=$(stat -c '%g' /var/run/docker.sock) export PHX_HOST="$rehearsal_phx_host" export PHX_SCHEME="$rehearsal_phx_scheme" export PHX_URL_PORT="$rehearsal_phx_url_port" export MAP_TILE_URL="$rehearsal_map_tile_url" export TRAEFIK_TRUSTED_IPS=127.0.0.1/32 export TRAEFIK_PROJECT_CONSTRAINT="$project" export TRAEFIK_APP_NAME="wnh-upgrade-rehearsal-$compact_id" export TRAEFIK_DOCKER_NETWORK="${project}_ingress" export REHEARSAL_IMAGE="who-need-help:upgrade-rehearsal-$run_id" compose() { docker compose \ --project-name "$project" \ --env-file "$ENV_FILE" \ --file "$ROOT/compose.yaml" \ --file "$ROOT/compose.upgrade-rehearsal.yaml" \ "$@" } cleanup() { status=$? trap - EXIT HUP INT TERM if [ "$status" -ne 0 ]; then compose ps --all >"$output_dir/compose-ps.txt" 2>&1 || true fi if ! compose down --volumes --remove-orphans >/dev/null 2>&1; then echo "Could not remove the isolated rehearsal Compose project." >&2 if [ "$status" -eq 0 ]; then status=1 fi fi if ! docker image rm "$REHEARSAL_IMAGE" >/dev/null 2>&1; then if docker image inspect "$REHEARSAL_IMAGE" >/dev/null 2>&1; then echo "Could not remove the isolated rehearsal image." >&2 if [ "$status" -eq 0 ]; then status=1 fi fi fi remaining_containers=$( docker ps --all --quiet \ --filter "label=com.docker.compose.project=$project" ) remaining_volumes=$( docker volume ls --quiet \ --filter "label=com.docker.compose.project=$project" ) remaining_networks=$( docker network ls --quiet \ --filter "label=com.docker.compose.project=$project" ) if [ -n "$remaining_containers$remaining_volumes$remaining_networks" ]; then echo "Isolated rehearsal resources remain after cleanup." >&2 status=1 fi exit "$status" } trap cleanup EXIT HUP INT TERM database_snapshot() { target=$1 tables=$( compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command " SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename NOT IN ('oban_jobs', 'schema_migrations', 'spatial_ref_sys') ORDER BY tablename; " ) : >"$target" for table in $tables; do row_count=$( compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command "SELECT count(*) FROM \"$table\";" ) printf '%s %s\n' "$table" "$row_count" >>"$target" done } compose config --quiet compose build migrate >"$output_dir/image-build.txt" compose up --detach --wait db compose exec -T db \ createdb \ --username "$POSTGRES_USER" \ --template template0 \ "$rehearsal_db" compose exec -T db \ pg_restore \ --username "$POSTGRES_USER" \ --dbname "$rehearsal_db" \ --exit-on-error \ --no-owner \ --no-privileges <"$dump" compose exec -T db pg_restore --list <"$dump" >"$output_dir/pg-restore-list.txt" database_snapshot "$output_dir/database-before.txt" compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command "SELECT version FROM schema_migrations ORDER BY version;" \ >"$output_dir/migrations-before.txt" compose up --detach --wait mailpit migrate proxy web worker database_snapshot "$output_dir/database-after.txt" diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \ >"$output_dir/database-diff.txt" expected_migrations=$( find "$ROOT/priv/repo/migrations" -type f -name '[0-9]*.exs' \ -exec basename {} \; | sed 's/_.*//' | sort >"$output_dir/migrations-expected.txt" wc -l <"$output_dir/migrations-expected.txt" | tr -d ' ' ) actual_migrations=$( compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command "SELECT count(*) FROM schema_migrations;" ) compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command "SELECT version FROM schema_migrations ORDER BY version;" \ >"$output_dir/migrations-after.txt" diff -u "$output_dir/migrations-expected.txt" "$output_dir/migrations-after.txt" \ >"$output_dir/migrations-diff.txt" description_columns=$( compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command " SELECT count(*) FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'categories' AND column_name = 'descriptions' AND is_nullable = 'NO'; " ) valid_cursor_indexes=$( compose exec -T db \ psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \ --tuples-only --no-align \ --command " SELECT count(*) FROM pg_class index_relation JOIN pg_index index_metadata ON index_metadata.indexrelid = index_relation.oid JOIN pg_namespace namespace ON namespace.oid = index_relation.relnamespace WHERE namespace.nspname = 'public' AND index_relation.relname IN ( 'help_requests_discovery_cursor_index', 'help_requests_requester_cursor_index', 'activities_discovery_cursor_index', 'messages_assignment_cursor_index', 'activity_messages_activity_cursor_index', 'reviews_visible_cursor_index', 'reports_cursor_index', 'blocks_blocker_cursor_index', 'abuse_signals_status_cursor_index', 'category_proposals_cursor_index', 'users_moderation_cursor_index' ) AND index_metadata.indisvalid AND index_metadata.indisready; " ) if [ "$actual_migrations" != "$expected_migrations" ]; then echo "Rehearsal migration count is $actual_migrations; expected $expected_migrations." >&2 exit 1 fi if [ "$description_columns" != "1" ] || [ "$valid_cursor_indexes" != "11" ]; then echo "The migrated rehearsal schema is incomplete." >&2 exit 1 fi proxy_port=$(compose port proxy 80 | sed 's/.*://') case "$proxy_port" in '' | *[!0-9]*) echo "Could not determine the isolated proxy port." >&2 exit 1 ;; esac web_container=$(compose ps --quiet web | head -n 1) if [ -z "$web_container" ]; then echo "No rehearsal web replica was found." >&2 exit 1 fi case "$PHX_SCHEME:$PHX_URL_PORT" in http:80 | https:443) public_origin="$PHX_SCHEME://$PHX_HOST" ;; *) public_origin="$PHX_SCHEME://$PHX_HOST:$PHX_URL_PORT" ;; esac { for path in / /safety; do header_file="$output_dir/redirect-$(printf '%s' "$path" | tr '/' '_').txt" status_code=$( curl \ --silent \ --show-error \ --dump-header "$header_file" \ --output /dev/null \ --write-out '%{http_code}' \ --header "Host: $PHX_HOST" \ "http://127.0.0.1:$proxy_port$path" ) location=$( awk 'tolower($1) == "location:" {sub(/\r$/, "", $2); print $2}' \ "$header_file" ) if [ "$status_code" != "301" ] || [ "$location" != "$public_origin$path" ]; then echo "Unexpected production redirect for $path: HTTP $status_code $location." >&2 exit 1 fi printf 'proxy %s %s %s\n' "$path" "$status_code" "$location" done for path in /healthz/live /healthz/ready; do status_code=$( curl \ --silent \ --show-error \ --output /dev/null \ --write-out '%{http_code}' \ --header "Host: $PHX_HOST" \ "http://127.0.0.1:$proxy_port$path" ) if [ "$status_code" != "200" ]; then echo "Unexpected HTTP $status_code for $path." >&2 exit 1 fi printf 'proxy %s %s\n' "$path" "$status_code" done for path in / /safety; do status_code=$( docker exec "$web_container" \ curl \ --silent \ --show-error \ --output /dev/null \ --write-out '%{http_code}' \ --header "Host: $PHX_HOST" \ --header "X-Forwarded-Proto: $PHX_SCHEME" \ --header "X-Forwarded-Port: $PHX_URL_PORT" \ "http://127.0.0.1:4000$path" ) if [ "$status_code" != "200" ]; then echo "Unexpected trusted-proxy HTTP $status_code for $path." >&2 exit 1 fi printf 'trusted-proxy %s %s\n' "$path" "$status_code" done } >"$output_dir/http.txt" docker exec "$web_container" /app/bin/who_need_help rpc ' nodes = Enum.sort([node() | Node.list()]) IO.inspect(%{nodes: nodes, count: length(nodes)}) if length(nodes) != 4, do: exit({:unexpected_cluster_size, nodes}) ' >"$output_dir/cluster.txt" COMPOSE_PROJECT_NAME="$project" \ "$ROOT/scripts/verify-realtime-cluster.sh" compose \ >"$output_dir/pubsub.txt" COMPOSE_PROJECT_NAME="$project" \ "$ROOT/scripts/verify-beam-runtime.sh" compose \ "$output_dir/beam-runtime.json" >/dev/null image_id=$(docker image inspect --format '{{.Id}}' "$REHEARSAL_IMAGE") source_commit=$(git rev-parse HEAD) cat >"$output_dir/summary.json" <