who_need_help/scripts/upgrade-rehearsal-compose.sh

#!/bin/sh
set -eu
umask 077

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"

if [ "$#" -ne 1 ]; then
  echo "Usage: $0 PATH_TO_CUSTOM_FORMAT_DUMP" >&2
  exit 1
fi

dump=$1
checksum="$dump.sha256"

if [ ! -f "$dump" ] || [ ! -s "$dump" ]; then
  echo "Dump does not exist or is empty: $dump" >&2
  exit 1
fi

if [ ! -f "$checksum" ]; then
  echo "Checksum manifest is required: $checksum" >&2
  exit 1
fi

"$ROOT/scripts/ensure-local-e2e-env.sh" >/dev/null

PUBLIC_ENV_FILE="$ROOT/.env"
ENV_FILE="$ROOT/.env.e2e"

if [ ! -f "$PUBLIC_ENV_FILE" ]; then
  echo "Missing $PUBLIC_ENV_FILE." >&2
  exit 1
fi

rehearsal_phx_host=$(
  sh -c '. "$1"; printf %s "${PHX_HOST:?PHX_HOST is missing from .env}"' \
    sh "$PUBLIC_ENV_FILE"
)
rehearsal_phx_scheme=$(
  sh -c '. "$1"; printf %s "${PHX_SCHEME:?PHX_SCHEME is missing from .env}"' \
    sh "$PUBLIC_ENV_FILE"
)
rehearsal_phx_url_port=$(
  sh -c '. "$1"; printf %s "${PHX_URL_PORT:?PHX_URL_PORT is missing from .env}"' \
    sh "$PUBLIC_ENV_FILE"
)
rehearsal_map_tile_url=$(
  sh -c \
    '. "$1"; printf %s "${MAP_TILE_URL:-https://tile.openstreetmap.org/{z}/{x}/{y}.png}"' \
    sh "$PUBLIC_ENV_FILE"
)

set -a
# shellcheck source=/dev/null
. "$ENV_FILE"
set +a

: "${POSTGRES_USER:?POSTGRES_USER is missing from .env.e2e}"
: "${DATABASE_URL:?DATABASE_URL is missing from .env.e2e}"

dump_dir=$(CDPATH='' cd -- "$(dirname -- "$dump")" && pwd)
dump_name=$(basename -- "$dump")

(
  cd "$dump_dir"
  sha256sum --check --status "$(basename -- "$checksum")"
)

run_id="$(date -u +%Y%m%d%H%M%S)-$$"
compact_id=$(printf '%s' "$run_id" | tr -d '-')
project="who_need_help_upgrade_rehearsal_$compact_id"
rehearsal_db="wnh_upgrade_rehearsal_$compact_id"
output_dir="$ROOT/output/upgrade-rehearsal/$run_id"

case "$project:$rehearsal_db" in
  who_need_help_upgrade_rehearsal_*:wnh_upgrade_rehearsal_*) ;;
  *)
    echo "Refusing unexpected rehearsal identifiers." >&2
    exit 1
    ;;
esac

mkdir -p "$output_dir"
chmod 700 "$ROOT/output" "$ROOT/output/upgrade-rehearsal" "$output_dir"

case "$DATABASE_URL" in
  *\?*)
    database_base=${DATABASE_URL%%\?*}
    database_query="?${DATABASE_URL#*\?}"
    ;;

  *)
    database_base=$DATABASE_URL
    database_query=
    ;;
esac

database_prefix=${database_base%/*}

if [ "$database_prefix" = "$database_base" ]; then
  echo "Could not derive the isolated rehearsal DATABASE_URL." >&2
  exit 1
fi

export DATABASE_URL="$database_prefix/$rehearsal_db$database_query"
export HTTP_PORT=0
export MAILPIT_PORT=0
export DOCKER_SOCKET_GID
DOCKER_SOCKET_GID=$(stat -c '%g' /var/run/docker.sock)
export PHX_HOST="$rehearsal_phx_host"
export PHX_SCHEME="$rehearsal_phx_scheme"
export PHX_URL_PORT="$rehearsal_phx_url_port"
export MAP_TILE_URL="$rehearsal_map_tile_url"
export TRAEFIK_TRUSTED_IPS=127.0.0.1/32
export TRAEFIK_PROJECT_CONSTRAINT="$project"
export TRAEFIK_APP_NAME="wnh-upgrade-rehearsal-$compact_id"
export TRAEFIK_DOCKER_NETWORK="${project}_ingress"
export REHEARSAL_IMAGE="who-need-help:upgrade-rehearsal-$run_id"

compose() {
  docker compose \
    --project-name "$project" \
    --env-file "$ENV_FILE" \
    --file "$ROOT/compose.yaml" \
    --file "$ROOT/compose.upgrade-rehearsal.yaml" \
    "$@"
}

cleanup() {
  status=$?
  trap - EXIT HUP INT TERM

  if [ "$status" -ne 0 ]; then
    compose ps --all >"$output_dir/compose-ps.txt" 2>&1 || true
  fi

  if ! compose down --volumes --remove-orphans >/dev/null 2>&1; then
    echo "Could not remove the isolated rehearsal Compose project." >&2
    if [ "$status" -eq 0 ]; then
      status=1
    fi
  fi

  if ! docker image rm "$REHEARSAL_IMAGE" >/dev/null 2>&1; then
    if docker image inspect "$REHEARSAL_IMAGE" >/dev/null 2>&1; then
      echo "Could not remove the isolated rehearsal image." >&2
      if [ "$status" -eq 0 ]; then
        status=1
      fi
    fi
  fi

  remaining_containers=$(
    docker ps --all --quiet \
      --filter "label=com.docker.compose.project=$project"
  )
  remaining_volumes=$(
    docker volume ls --quiet \
      --filter "label=com.docker.compose.project=$project"
  )
  remaining_networks=$(
    docker network ls --quiet \
      --filter "label=com.docker.compose.project=$project"
  )

  if [ -n "$remaining_containers$remaining_volumes$remaining_networks" ]; then
    echo "Isolated rehearsal resources remain after cleanup." >&2
    status=1
  fi

  exit "$status"
}
trap cleanup EXIT HUP INT TERM

database_snapshot() {
  target=$1
  tables=$(
    compose exec -T db \
      psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
      --tuples-only --no-align \
      --command "
        SELECT tablename
        FROM pg_tables
        WHERE schemaname = 'public'
          AND tablename NOT IN ('oban_jobs', 'schema_migrations', 'spatial_ref_sys')
        ORDER BY tablename;
      "
  )

  : >"$target"

  for table in $tables; do
    row_count=$(
      compose exec -T db \
        psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
        --tuples-only --no-align \
        --command "SELECT count(*) FROM \"$table\";"
    )
    printf '%s %s\n' "$table" "$row_count" >>"$target"
  done
}

compose config --quiet
compose build migrate >"$output_dir/image-build.txt"
compose up --detach --wait db

compose exec -T db \
  createdb \
  --username "$POSTGRES_USER" \
  --template template0 \
  "$rehearsal_db"

compose exec -T db \
  pg_restore \
  --username "$POSTGRES_USER" \
  --dbname "$rehearsal_db" \
  --exit-on-error \
  --no-owner \
  --no-privileges <"$dump"

compose exec -T db pg_restore --list <"$dump" >"$output_dir/pg-restore-list.txt"

database_snapshot "$output_dir/database-before.txt"
compose exec -T db \
  psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
  --tuples-only --no-align \
  --command "SELECT version FROM schema_migrations ORDER BY version;" \
  >"$output_dir/migrations-before.txt"

compose up --detach --wait mailpit migrate proxy web worker

database_snapshot "$output_dir/database-after.txt"
diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \
  >"$output_dir/database-diff.txt"

expected_migrations=$(
  find "$ROOT/priv/repo/migrations" -type f -name '[0-9]*.exs' \
    -exec basename {} \; |
    sed 's/_.*//' |
    sort >"$output_dir/migrations-expected.txt"
  wc -l <"$output_dir/migrations-expected.txt" | tr -d ' '
)
actual_migrations=$(
  compose exec -T db \
    psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
    --tuples-only --no-align \
    --command "SELECT count(*) FROM schema_migrations;"
)
compose exec -T db \
  psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
  --tuples-only --no-align \
  --command "SELECT version FROM schema_migrations ORDER BY version;" \
  >"$output_dir/migrations-after.txt"
diff -u "$output_dir/migrations-expected.txt" "$output_dir/migrations-after.txt" \
  >"$output_dir/migrations-diff.txt"

description_columns=$(
  compose exec -T db \
    psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
    --tuples-only --no-align \
    --command "
      SELECT count(*)
      FROM information_schema.columns
      WHERE table_schema = 'public'
        AND table_name = 'categories'
        AND column_name = 'descriptions'
        AND is_nullable = 'NO';
    "
)
valid_cursor_indexes=$(
  compose exec -T db \
    psql --username "$POSTGRES_USER" --dbname "$rehearsal_db" \
    --tuples-only --no-align \
    --command "
      SELECT count(*)
      FROM pg_class index_relation
      JOIN pg_index index_metadata
        ON index_metadata.indexrelid = index_relation.oid
      JOIN pg_namespace namespace
        ON namespace.oid = index_relation.relnamespace
      WHERE namespace.nspname = 'public'
        AND index_relation.relname IN (
          'help_requests_discovery_cursor_index',
          'help_requests_requester_cursor_index',
          'activities_discovery_cursor_index',
          'messages_assignment_cursor_index',
          'activity_messages_activity_cursor_index',
          'reviews_visible_cursor_index',
          'reports_cursor_index',
          'blocks_blocker_cursor_index',
          'abuse_signals_status_cursor_index',
          'category_proposals_cursor_index',
          'users_moderation_cursor_index'
        )
        AND index_metadata.indisvalid
        AND index_metadata.indisready;
    "
)

if [ "$actual_migrations" != "$expected_migrations" ]; then
  echo "Rehearsal migration count is $actual_migrations; expected $expected_migrations." >&2
  exit 1
fi

if [ "$description_columns" != "1" ] || [ "$valid_cursor_indexes" != "11" ]; then
  echo "The migrated rehearsal schema is incomplete." >&2
  exit 1
fi

proxy_port=$(compose port proxy 80 | sed 's/.*://')

case "$proxy_port" in
  '' | *[!0-9]*)
    echo "Could not determine the isolated proxy port." >&2
    exit 1
    ;;
esac

web_container=$(compose ps --quiet web | head -n 1)

if [ -z "$web_container" ]; then
  echo "No rehearsal web replica was found." >&2
  exit 1
fi

case "$PHX_SCHEME:$PHX_URL_PORT" in
  http:80 | https:443)
    public_origin="$PHX_SCHEME://$PHX_HOST"
    ;;

  *)
    public_origin="$PHX_SCHEME://$PHX_HOST:$PHX_URL_PORT"
    ;;
esac

{
  for path in / /safety; do
    header_file="$output_dir/redirect-$(printf '%s' "$path" | tr '/' '_').txt"
    status_code=$(
      curl \
        --silent \
        --show-error \
        --dump-header "$header_file" \
        --output /dev/null \
        --write-out '%{http_code}' \
        --header "Host: $PHX_HOST" \
        "http://127.0.0.1:$proxy_port$path"
    )

    location=$(
      awk 'tolower($1) == "location:" {sub(/\r$/, "", $2); print $2}' \
        "$header_file"
    )

    if [ "$status_code" != "301" ] || [ "$location" != "$public_origin$path" ]; then
      echo "Unexpected production redirect for $path: HTTP $status_code $location." >&2
      exit 1
    fi

    printf 'proxy %s %s %s\n' "$path" "$status_code" "$location"
  done

  for path in /healthz/live /healthz/ready; do
    status_code=$(
      curl \
        --silent \
        --show-error \
        --output /dev/null \
        --write-out '%{http_code}' \
        --header "Host: $PHX_HOST" \
        "http://127.0.0.1:$proxy_port$path"
    )

    if [ "$status_code" != "200" ]; then
      echo "Unexpected HTTP $status_code for $path." >&2
      exit 1
    fi

    printf 'proxy %s %s\n' "$path" "$status_code"
  done

  for path in / /safety; do
    status_code=$(
      docker exec "$web_container" \
        curl \
        --silent \
        --show-error \
        --output /dev/null \
        --write-out '%{http_code}' \
        --header "Host: $PHX_HOST" \
        --header "X-Forwarded-Proto: $PHX_SCHEME" \
        --header "X-Forwarded-Port: $PHX_URL_PORT" \
        "http://127.0.0.1:4000$path"
    )

    if [ "$status_code" != "200" ]; then
      echo "Unexpected trusted-proxy HTTP $status_code for $path." >&2
      exit 1
    fi

    printf 'trusted-proxy %s %s\n' "$path" "$status_code"
  done
} >"$output_dir/http.txt"

docker exec "$web_container" /app/bin/who_need_help rpc '
  nodes = Enum.sort([node() | Node.list()])
  IO.inspect(%{nodes: nodes, count: length(nodes)})
  if length(nodes) != 4, do: exit({:unexpected_cluster_size, nodes})
' >"$output_dir/cluster.txt"

COMPOSE_PROJECT_NAME="$project" \
  "$ROOT/scripts/verify-realtime-cluster.sh" compose \
  >"$output_dir/pubsub.txt"
COMPOSE_PROJECT_NAME="$project" \
  "$ROOT/scripts/verify-beam-runtime.sh" compose \
  "$output_dir/beam-runtime.json" >/dev/null

image_id=$(docker image inspect --format '{{.Id}}' "$REHEARSAL_IMAGE")
source_commit=$(git rev-parse HEAD)

cat >"$output_dir/summary.json" <<EOF
{
  "status": "passed",
  "source_commit": "$source_commit",
  "image_id": "$image_id",
  "input_dump": "$dump_name",
  "expected_migrations": $expected_migrations,
  "actual_migrations": $actual_migrations,
  "migration_version_diff_bytes": 0,
  "valid_cursor_indexes": $valid_cursor_indexes,
  "application_table_diff_bytes": 0,
  "web_replicas": 2,
  "worker_replicas": 2,
  "cluster_nodes": 4,
  "pubsub": "passed",
  "http": "passed"
}
EOF

chmod 600 "$output_dir"/*

echo "Isolated Compose upgrade rehearsal passed."
echo "Evidence: $output_dir"