who_need_help/scripts/clean-deploy-verify.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"

for command in curl docker git jq openssl tar; do
  if ! command -v "$command" >/dev/null 2>&1; then
    echo "Required command is unavailable: $command" >&2
    exit 1
  fi
done

run_id="$(date -u +%Y%m%d%H%M%S)-$$-$(openssl rand -hex 3)"
compact_id=${run_id//-/}
project="wnh_portability_$compact_id"
image="who-need-help:portability-$run_id"
output_dir="$ROOT/output/portability/$run_id"
workspace=$(mktemp -d "${TMPDIR:-/tmp}/wnh-portability.$run_id.XXXXXX")
env_file="$workspace/.env"

case "$project:$image:$workspace" in
  wnh_portability_*:who-need-help:portability-*:"${TMPDIR:-/tmp}"/wnh-portability.*) ;;
  *)
    echo "Refusing unexpected portability-drill identifiers." >&2
    exit 1
    ;;
esac

mkdir -p "$output_dir"
chmod 700 "$ROOT/output" "$ROOT/output/portability" "$output_dir"

compose=(
  docker compose
  --project-name "$project"
  --project-directory "$workspace"
  --env-file "$env_file"
  --file "$workspace/compose.yaml"
  --file "$workspace/compose.portability.yaml"
)

cleanup() {
  local status=$?
  local cleanup_status=0

  trap - EXIT HUP INT TERM

  if [[ -f "$env_file" ]]; then
    if [[ "$status" -ne 0 ]]; then
      "${compose[@]}" ps --all >"$output_dir/compose-ps-on-failure.txt" 2>&1 || true
      "${compose[@]}" logs --no-color >"$output_dir/logs-on-failure.txt" 2>&1 || true
    fi

    if ! "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1; then
      echo "Could not remove the isolated portability Compose project." >&2
      cleanup_status=1
    fi
  fi

  if docker image inspect "$image" >/dev/null 2>&1; then
    if ! docker image rm "$image" >/dev/null 2>&1; then
      echo "Could not remove the isolated portability image." >&2
      cleanup_status=1
    fi
  fi

  remaining_containers=$(
    docker ps --all --quiet \
      --filter "label=com.docker.compose.project=$project"
  )
  remaining_volumes=$(
    docker volume ls --quiet \
      --filter "label=com.docker.compose.project=$project"
  )
  remaining_networks=$(
    docker network ls --quiet \
      --filter "label=com.docker.compose.project=$project"
  )

  if [[ -n "$remaining_containers$remaining_volumes$remaining_networks" ]]; then
    echo "Isolated portability resources remain after cleanup." >&2
    cleanup_status=1
  fi

  if [[ "$workspace" == "${TMPDIR:-/tmp}"/wnh-portability.* ]]; then
    rm -rf -- "$workspace"
  else
    echo "Refusing to remove an unexpected workspace path: $workspace" >&2
    cleanup_status=1
  fi

  if [[ "$cleanup_status" -eq 0 ]]; then
    printf 'project_resources=absent\nimage=absent\nworkspace=absent\n' \
      >"$output_dir/cleanup.txt"
  fi

  if [[ "$status" -eq 0 && "$cleanup_status" -ne 0 ]]; then
    status=1
  fi

  exit "$status"
}
trap cleanup EXIT HUP INT TERM

if [[ -n "$(
  docker ps --all --quiet \
    --filter "label=com.docker.compose.project=$project"
)" ]] ||
  [[ -n "$(
    docker volume ls --quiet \
      --filter "label=com.docker.compose.project=$project"
  )" ]] ||
  [[ -n "$(
    docker network ls --quiet \
      --filter "label=com.docker.compose.project=$project"
  )" ]] ||
  docker image inspect "$image" >/dev/null 2>&1; then
  echo "The generated portability identifiers already exist." >&2
  exit 1
fi

source_commit=$(git rev-parse --verify HEAD)
git archive --format=tar "$source_commit" | tar -xf - -C "$workspace"

if [[ -e "$workspace/.git" || -e "$workspace/output" || -e "$workspace/.env.load" ||
  -e "$workspace/.env.e2e" ]]; then
  echo "The tracked archive unexpectedly contains local state." >&2
  exit 1
fi

postgres_user="wnh_port_$(openssl rand -hex 6)"
postgres_db="wnh_port_$(openssl rand -hex 6)"
postgres_password=$(openssl rand -hex 32)
secret_key_base=$(openssl rand -hex 64)
handover_secret=$(openssl rand -hex 64)
release_cookie=$(openssl rand -hex 64)
metrics_token=$(openssl rand -hex 32)
docker_socket_gid=$(stat -c '%g' /var/run/docker.sock)

cat >"$env_file" <<EOF
HTTP_PORT=0
MAILPIT_PORT=0
DOCKER_SOCKET_GID=$docker_socket_gid
TRAEFIK_TRUSTED_IPS=127.0.0.1/32
TRAEFIK_RETRY_ATTEMPTS=3
TRAEFIK_PROJECT_CONSTRAINT=$project
TRAEFIK_APP_NAME=wnh-portability-$compact_id
TRAEFIK_DOCKER_NETWORK=${project}_ingress
TRAEFIK_ROUTER_RULE='Host(\`portability.local\`)'
PHX_HOST=portability.local
PHX_SCHEME=http
PHX_URL_PORT=80
MAP_TILE_URL=https://tile.openstreetmap.org/{z}/{x}/{y}.png
GITHUB_OAUTH_CLIENT_ID=
GITHUB_OAUTH_CLIENT_SECRET=
PUSH_HTTP_ENDPOINT=
PUSH_HTTP_BEARER_TOKEN=
POSTGRES_DB=$postgres_db
POSTGRES_USER=$postgres_user
POSTGRES_PASSWORD=$postgres_password
DATABASE_URL=ecto://$postgres_user:$postgres_password@db/$postgres_db
WEB_POOL_SIZE=4
WORKER_POOL_SIZE=2
MIGRATE_POOL_SIZE=2
OBAN_MAINTENANCE_CONCURRENCY=2
OBAN_PUSH_CONCURRENCY=1
WEB_REPLICAS=2
WORKER_REPLICAS=2
ERLANG_PORT_LIMIT=65536
SECRET_KEY_BASE=$secret_key_base
HANDOVER_SECRET=$handover_secret
RELEASE_COOKIE=$release_cookie
METRICS_TOKEN=$metrics_token
SMTP_RELAY=mailpit
SMTP_PORT=1025
SMTP_USERNAME=
SMTP_PASSWORD=
SMTP_AUTH=never
SMTP_TLS=never
SMTP_SSL=false
EMAIL_FROM_NAME="Who Need Help portability drill"
EMAIL_FROM_ADDRESS=portability@example.invalid
CODEX_SESSION_ID=local-portability-$run_id
RATE_LIMIT_POLICIES_JSON={}
PORTABILITY_IMAGE=$image
EOF
chmod 600 "$env_file"
unset postgres_password secret_key_base handover_secret release_cookie metrics_token

{
  printf 'source_commit=%s\n' "$source_commit"
  printf 'project=%s\n' "$project"
  printf 'image=%s\n' "$image"
  docker version --format \
    'docker_server={{.Server.Version}} docker_client={{.Client.Version}}'
  docker compose version
} >"$output_dir/environment.txt"

"${compose[@]}" config --quiet
printf 'tracked_archive_config=valid\n' >"$output_dir/config.txt"

"${compose[@]}" up --detach --wait --build
"${compose[@]}" ps --all --format json >"$output_dir/compose-ps.jsonl"

web_count=$("${compose[@]}" ps --status running --quiet web | wc -l)
worker_count=$("${compose[@]}" ps --status running --quiet worker | wc -l)
db_count=$("${compose[@]}" ps --status running --quiet db | wc -l)
proxy_count=$("${compose[@]}" ps --status running --quiet proxy | wc -l)
mailpit_count=$("${compose[@]}" ps --status running --quiet mailpit | wc -l)
migrate_exit=$(
  "${compose[@]}" ps --all --format json migrate |
    jq -sr 'if length == 1 then .[0].ExitCode else null end'
)

if [[ "$web_count" -ne 2 || "$worker_count" -ne 2 ||
  "$db_count" -ne 1 || "$proxy_count" -ne 1 ||
  "$mailpit_count" -ne 1 || "$migrate_exit" != "0" ]]; then
  echo "The clean deployment did not produce the expected topology." >&2
  exit 1
fi

mapfile -t app_containers < <("${compose[@]}" ps --quiet web worker)
if [[ "${#app_containers[@]}" -ne 4 ]]; then
  echo "Expected four running application containers." >&2
  exit 1
fi

expected_image_id=$(docker image inspect "$image" --format '{{.Id}}')
for container in "${app_containers[@]}"; do
  actual_image_id=$(docker inspect "$container" --format '{{.Image}}')
  if [[ "$actual_image_id" != "$expected_image_id" ]]; then
    echo "An application container is not using the archived-source image." >&2
    exit 1
  fi
done
printf '%s\n' "$expected_image_id" >"$output_dir/application-image-id.txt"

expected_migrations=$(
  find "$workspace/priv/repo/migrations" -type f -name '[0-9]*.exs' |
    wc -l
)
"${compose[@]}" exec -T db \
  psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
  --username "$postgres_user" --dbname "$postgres_db" \
  --command "SELECT version FROM schema_migrations ORDER BY version;" \
  >"$output_dir/migrations-before-repeat.txt"
actual_migrations=$(wc -l <"$output_dir/migrations-before-repeat.txt")

if [[ "$actual_migrations" -ne "$expected_migrations" ]]; then
  echo "Fresh migration count is $actual_migrations; expected $expected_migrations." >&2
  exit 1
fi

category_count_before=$(
  "${compose[@]}" exec -T db \
    psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
    --username "$postgres_user" --dbname "$postgres_db" \
    --command "SELECT count(*) FROM categories;"
)

if [[ "$category_count_before" -le 0 ]]; then
  echo "Fresh migration did not seed any product category." >&2
  exit 1
fi

"${compose[@]}" run --rm migrate >"$output_dir/repeated-migrate.txt"

"${compose[@]}" exec -T db \
  psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
  --username "$postgres_user" --dbname "$postgres_db" \
  --command "SELECT version FROM schema_migrations ORDER BY version;" \
  >"$output_dir/migrations-after-repeat.txt"
category_count_after=$(
  "${compose[@]}" exec -T db \
    psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
    --username "$postgres_user" --dbname "$postgres_db" \
    --command "SELECT count(*) FROM categories;"
)

if ! diff -u "$output_dir/migrations-before-repeat.txt" \
  "$output_dir/migrations-after-repeat.txt" >"$output_dir/migrations-repeat.diff" ||
  [[ "$category_count_before" -ne "$category_count_after" ]]; then
  echo "The repeated migration changed migration or category counts." >&2
  exit 1
fi

{
  printf 'expected_migrations=%s\n' "$expected_migrations"
  printf 'actual_migrations=%s\n' "$actual_migrations"
  printf 'categories_before_repeat=%s\n' "$category_count_before"
  printf 'categories_after_repeat=%s\n' "$category_count_after"
} >"$output_dir/database.txt"

published=$("${compose[@]}" port proxy 80 | head -n 1)
http_port=${published##*:}
mailpit_published=$("${compose[@]}" port mailpit 8025 | head -n 1)
mailpit_port=${mailpit_published##*:}

if [[ ! "$http_port" =~ ^[1-9][0-9]*$ ||
  ! "$mailpit_port" =~ ^[1-9][0-9]*$ ]]; then
  echo "Compose did not publish dynamic local ports." >&2
  exit 1
fi

curl --fail-with-body --silent --show-error \
  --header 'Host: portability.local' \
  "http://127.0.0.1:$http_port/healthz/ready" \
  >"$output_dir/readiness.json"
curl --fail-with-body --silent --show-error \
  --header 'Host: portability.local' \
  "http://127.0.0.1:$http_port/" \
  >"$output_dir/home.html"
curl --fail-with-body --silent --show-error \
  --header 'Host: portability.local' \
  "http://127.0.0.1:$http_port/users/register" \
  >"$output_dir/register.html"
curl --fail-with-body --silent --show-error \
  "http://127.0.0.1:$mailpit_port/api/v1/info" \
  >"$output_dir/mailpit.json"

(
  cd "$workspace"
  COMPOSE_PROJECT_NAME="$project" \
    ./scripts/verify-realtime-cluster.sh compose
) >"$output_dir/pubsub-probe.txt"
(
  cd "$workspace"
  COMPOSE_PROJECT_NAME="$project" \
    ./scripts/verify-beam-runtime.sh compose \
    "$output_dir/beam-runtime.json"
) >/dev/null

"${compose[@]}" exec -T web /app/bin/who_need_help rpc '
  nodes = [node() | Node.list()] |> Enum.sort()
  result = %{node_count: length(nodes), nodes: nodes}
  IO.inspect(result)
  if result.node_count != 4, do: exit({:unexpected_cluster_size, result})
' >"$output_dir/cluster-size.txt"

docker inspect "${app_containers[@]}" |
  jq '[
    .[] | {
      name: .Name,
      image: .Image,
      restart_count: .RestartCount,
      oom_killed: .State.OOMKilled,
      status: .State.Status,
      health: (.State.Health.Status // null)
    }
  ]' >"$output_dir/application-containers.json"

if ! jq -e '
  length == 4 and
  all(
    .status == "running" and
    .restart_count == 0 and
    .oom_killed == false and
    (.health == null or .health == "healthy")
  )
' "$output_dir/application-containers.json" >/dev/null; then
  echo "A clean-deployment application container is unhealthy." >&2
  exit 1
fi

printf '%s\n' \
  "tracked archive clean deployment passed: $project" \
  "2 web + 2 worker replicas, current migrations, repeated migrate, HTTP, Mailpit, cluster, and PubSub passed"