test: verify clean Compose portability
This commit is contained in:
parent
b5a5aba202
commit
f5c188da36
14
README.md
14
README.md
|
|
@ -104,6 +104,20 @@ current migrations, starts 2 web and 2 worker replicas, checks the configured
|
||||||
public-origin proxy behavior and cross-node PubSub, compares application-table
|
public-origin proxy behavior and cross-node PubSub, compares application-table
|
||||||
counts, and removes its containers, networks, volume, and one-run image.
|
counts, and removes its containers, networks, volume, and one-run image.
|
||||||
|
|
||||||
|
Verify that the tracked Git revision can deploy from a clean directory with a
|
||||||
|
new environment, independent generated secrets, dynamic ports, and no
|
||||||
|
pre-existing image or volume:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/clean-deploy-verify.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The drill starts an isolated 2-web/2-worker Compose project, verifies current
|
||||||
|
and repeated migrations, HTTP and Mailpit routing, the four-node BEAM cluster,
|
||||||
|
and cross-replica PubSub, then removes its exact project, volumes, image, and
|
||||||
|
temporary tracked-file archive. It does not read or change the ordinary
|
||||||
|
Compose `.env` or database.
|
||||||
|
|
||||||
For the reproducible encrypted S3-compatible drill, first start the isolated
|
For the reproducible encrypted S3-compatible drill, first start the isolated
|
||||||
load project and then run:
|
load project and then run:
|
||||||
|
|
||||||
|
|
|
||||||
12
compose.portability.yaml
Normal file
12
compose.portability.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
services:
|
||||||
|
migrate:
|
||||||
|
image: ${PORTABILITY_IMAGE:?Set PORTABILITY_IMAGE for the isolated portability drill}
|
||||||
|
pull_policy: never
|
||||||
|
|
||||||
|
web:
|
||||||
|
image: ${PORTABILITY_IMAGE:?Set PORTABILITY_IMAGE for the isolated portability drill}
|
||||||
|
pull_policy: never
|
||||||
|
|
||||||
|
worker:
|
||||||
|
image: ${PORTABILITY_IMAGE:?Set PORTABILITY_IMAGE for the isolated portability drill}
|
||||||
|
pull_policy: never
|
||||||
|
|
@ -193,6 +193,35 @@ containers are outside the generated project scope. Non-secret evidence is
|
||||||
retained under ignored mode-`0700`
|
retained under ignored mode-`0700`
|
||||||
`output/upgrade-rehearsal/<run-id>/`, with files mode `0600`.
|
`output/upgrade-rehearsal/<run-id>/`, with files mode `0600`.
|
||||||
|
|
||||||
|
## Clean tracked-revision deployment drill
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/clean-deploy-verify.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The script archives the current tracked Git revision into a new temporary
|
||||||
|
directory. It verifies that the archive contains no local `.env`, Git
|
||||||
|
metadata, generated output, or existing E2E/load environment. It generates
|
||||||
|
independent one-run PostgreSQL, Phoenix, handover, BEAM-cookie, and metrics
|
||||||
|
secrets with mode `0600`; optional GitHub OAuth and push delivery stay
|
||||||
|
disabled.
|
||||||
|
|
||||||
|
The drill uses a unique Compose project, application image, router/service
|
||||||
|
name, internal network, PostgreSQL volume, database, and dynamic host ports.
|
||||||
|
It requires exactly two healthy web and two running worker replicas, all
|
||||||
|
tracked migrations, seeded categories, an idempotent repeated migration,
|
||||||
|
working readiness/home/registration/Mailpit routes, four connected BEAM
|
||||||
|
nodes, and a cross-replica PubSub message. It then removes the exact project
|
||||||
|
including volumes, its one-run image, and the temporary archive, and verifies
|
||||||
|
that those scoped resources are absent. Evidence is retained under
|
||||||
|
`output/portability/<run-id>/`.
|
||||||
|
|
||||||
|
This is a local Docker portability observation. It does not establish
|
||||||
|
production SMTP, OAuth, push-provider, TLS, database-HA, storage, capacity, or
|
||||||
|
jurisdictional readiness.
|
||||||
|
|
||||||
## Service checks
|
## Service checks
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
353
scripts/clean-deploy-verify.sh
Executable file
353
scripts/clean-deploy-verify.sh
Executable file
|
|
@ -0,0 +1,353 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
for command in curl docker git jq openssl tar; do
|
||||||
|
if ! command -v "$command" >/dev/null 2>&1; then
|
||||||
|
echo "Required command is unavailable: $command" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
run_id="$(date -u +%Y%m%d%H%M%S)-$$-$(openssl rand -hex 3)"
|
||||||
|
compact_id=${run_id//-/}
|
||||||
|
project="wnh_portability_$compact_id"
|
||||||
|
image="who-need-help:portability-$run_id"
|
||||||
|
output_dir="$ROOT/output/portability/$run_id"
|
||||||
|
workspace=$(mktemp -d "${TMPDIR:-/tmp}/wnh-portability.$run_id.XXXXXX")
|
||||||
|
env_file="$workspace/.env"
|
||||||
|
|
||||||
|
case "$project:$image:$workspace" in
|
||||||
|
wnh_portability_*:who-need-help:portability-*:"${TMPDIR:-/tmp}"/wnh-portability.*) ;;
|
||||||
|
*)
|
||||||
|
echo "Refusing unexpected portability-drill identifiers." >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
mkdir -p "$output_dir"
|
||||||
|
chmod 700 "$ROOT/output" "$ROOT/output/portability" "$output_dir"
|
||||||
|
|
||||||
|
compose=(
|
||||||
|
docker compose
|
||||||
|
--project-name "$project"
|
||||||
|
--project-directory "$workspace"
|
||||||
|
--env-file "$env_file"
|
||||||
|
--file "$workspace/compose.yaml"
|
||||||
|
--file "$workspace/compose.portability.yaml"
|
||||||
|
)
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local status=$?
|
||||||
|
local cleanup_status=0
|
||||||
|
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
|
||||||
|
if [[ -f "$env_file" ]]; then
|
||||||
|
if [[ "$status" -ne 0 ]]; then
|
||||||
|
"${compose[@]}" ps --all >"$output_dir/compose-ps-on-failure.txt" 2>&1 || true
|
||||||
|
"${compose[@]}" logs --no-color >"$output_dir/logs-on-failure.txt" 2>&1 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1; then
|
||||||
|
echo "Could not remove the isolated portability Compose project." >&2
|
||||||
|
cleanup_status=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if docker image inspect "$image" >/dev/null 2>&1; then
|
||||||
|
if ! docker image rm "$image" >/dev/null 2>&1; then
|
||||||
|
echo "Could not remove the isolated portability image." >&2
|
||||||
|
cleanup_status=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
remaining_containers=$(
|
||||||
|
docker ps --all --quiet \
|
||||||
|
--filter "label=com.docker.compose.project=$project"
|
||||||
|
)
|
||||||
|
remaining_volumes=$(
|
||||||
|
docker volume ls --quiet \
|
||||||
|
--filter "label=com.docker.compose.project=$project"
|
||||||
|
)
|
||||||
|
remaining_networks=$(
|
||||||
|
docker network ls --quiet \
|
||||||
|
--filter "label=com.docker.compose.project=$project"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ -n "$remaining_containers$remaining_volumes$remaining_networks" ]]; then
|
||||||
|
echo "Isolated portability resources remain after cleanup." >&2
|
||||||
|
cleanup_status=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$workspace" == "${TMPDIR:-/tmp}"/wnh-portability.* ]]; then
|
||||||
|
rm -rf -- "$workspace"
|
||||||
|
else
|
||||||
|
echo "Refusing to remove an unexpected workspace path: $workspace" >&2
|
||||||
|
cleanup_status=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$cleanup_status" -eq 0 ]]; then
|
||||||
|
printf 'project_resources=absent\nimage=absent\nworkspace=absent\n' \
|
||||||
|
>"$output_dir/cleanup.txt"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$status" -eq 0 && "$cleanup_status" -ne 0 ]]; then
|
||||||
|
status=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT HUP INT TERM
|
||||||
|
|
||||||
|
if [[ -n "$(
|
||||||
|
docker ps --all --quiet \
|
||||||
|
--filter "label=com.docker.compose.project=$project"
|
||||||
|
)" ]] ||
|
||||||
|
[[ -n "$(
|
||||||
|
docker volume ls --quiet \
|
||||||
|
--filter "label=com.docker.compose.project=$project"
|
||||||
|
)" ]] ||
|
||||||
|
[[ -n "$(
|
||||||
|
docker network ls --quiet \
|
||||||
|
--filter "label=com.docker.compose.project=$project"
|
||||||
|
)" ]] ||
|
||||||
|
docker image inspect "$image" >/dev/null 2>&1; then
|
||||||
|
echo "The generated portability identifiers already exist." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
source_commit=$(git rev-parse --verify HEAD)
|
||||||
|
git archive --format=tar "$source_commit" | tar -xf - -C "$workspace"
|
||||||
|
|
||||||
|
if [[ -e "$workspace/.git" || -e "$workspace/output" || -e "$workspace/.env.load" ||
|
||||||
|
-e "$workspace/.env.e2e" ]]; then
|
||||||
|
echo "The tracked archive unexpectedly contains local state." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
postgres_user="wnh_port_$(openssl rand -hex 6)"
|
||||||
|
postgres_db="wnh_port_$(openssl rand -hex 6)"
|
||||||
|
postgres_password=$(openssl rand -hex 32)
|
||||||
|
secret_key_base=$(openssl rand -hex 64)
|
||||||
|
handover_secret=$(openssl rand -hex 64)
|
||||||
|
release_cookie=$(openssl rand -hex 64)
|
||||||
|
metrics_token=$(openssl rand -hex 32)
|
||||||
|
|
||||||
|
cat >"$env_file" <<EOF
|
||||||
|
HTTP_PORT=0
|
||||||
|
MAILPIT_PORT=0
|
||||||
|
TRAEFIK_TRUSTED_IPS=127.0.0.1/32
|
||||||
|
TRAEFIK_RETRY_ATTEMPTS=3
|
||||||
|
TRAEFIK_PROJECT_CONSTRAINT=$project
|
||||||
|
TRAEFIK_APP_NAME=wnh-portability-$compact_id
|
||||||
|
TRAEFIK_DOCKER_NETWORK=${project}_internal
|
||||||
|
TRAEFIK_ROUTER_RULE='Host(\`portability.local\`)'
|
||||||
|
PHX_HOST=portability.local
|
||||||
|
PHX_SCHEME=http
|
||||||
|
PHX_URL_PORT=80
|
||||||
|
MAP_TILE_URL=https://tile.openstreetmap.org/{z}/{x}/{y}.png
|
||||||
|
GITHUB_OAUTH_CLIENT_ID=
|
||||||
|
GITHUB_OAUTH_CLIENT_SECRET=
|
||||||
|
PUSH_HTTP_ENDPOINT=
|
||||||
|
PUSH_HTTP_BEARER_TOKEN=
|
||||||
|
POSTGRES_DB=$postgres_db
|
||||||
|
POSTGRES_USER=$postgres_user
|
||||||
|
POSTGRES_PASSWORD=$postgres_password
|
||||||
|
DATABASE_URL=ecto://$postgres_user:$postgres_password@db/$postgres_db
|
||||||
|
POOL_SIZE=10
|
||||||
|
SECRET_KEY_BASE=$secret_key_base
|
||||||
|
HANDOVER_SECRET=$handover_secret
|
||||||
|
RELEASE_COOKIE=$release_cookie
|
||||||
|
METRICS_TOKEN=$metrics_token
|
||||||
|
SMTP_RELAY=mailpit
|
||||||
|
SMTP_PORT=1025
|
||||||
|
SMTP_USERNAME=
|
||||||
|
SMTP_PASSWORD=
|
||||||
|
SMTP_AUTH=never
|
||||||
|
SMTP_TLS=never
|
||||||
|
SMTP_SSL=false
|
||||||
|
EMAIL_FROM_NAME="Who Need Help portability drill"
|
||||||
|
EMAIL_FROM_ADDRESS=portability@example.invalid
|
||||||
|
CODEX_SESSION_ID=local-portability-$run_id
|
||||||
|
RATE_LIMIT_POLICIES_JSON={}
|
||||||
|
PORTABILITY_IMAGE=$image
|
||||||
|
EOF
|
||||||
|
chmod 600 "$env_file"
|
||||||
|
unset postgres_password secret_key_base handover_secret release_cookie metrics_token
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'source_commit=%s\n' "$source_commit"
|
||||||
|
printf 'project=%s\n' "$project"
|
||||||
|
printf 'image=%s\n' "$image"
|
||||||
|
docker version --format \
|
||||||
|
'docker_server={{.Server.Version}} docker_client={{.Client.Version}}'
|
||||||
|
docker compose version
|
||||||
|
} >"$output_dir/environment.txt"
|
||||||
|
|
||||||
|
"${compose[@]}" config --quiet
|
||||||
|
printf 'tracked_archive_config=valid\n' >"$output_dir/config.txt"
|
||||||
|
|
||||||
|
"${compose[@]}" up --detach --wait --build
|
||||||
|
"${compose[@]}" ps --all --format json >"$output_dir/compose-ps.jsonl"
|
||||||
|
|
||||||
|
web_count=$("${compose[@]}" ps --status running --quiet web | wc -l)
|
||||||
|
worker_count=$("${compose[@]}" ps --status running --quiet worker | wc -l)
|
||||||
|
db_count=$("${compose[@]}" ps --status running --quiet db | wc -l)
|
||||||
|
proxy_count=$("${compose[@]}" ps --status running --quiet proxy | wc -l)
|
||||||
|
mailpit_count=$("${compose[@]}" ps --status running --quiet mailpit | wc -l)
|
||||||
|
migrate_exit=$(
|
||||||
|
"${compose[@]}" ps --all --format json migrate |
|
||||||
|
jq -sr 'if length == 1 then .[0].ExitCode else null end'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$web_count" -ne 2 || "$worker_count" -ne 2 ||
|
||||||
|
"$db_count" -ne 1 || "$proxy_count" -ne 1 ||
|
||||||
|
"$mailpit_count" -ne 1 || "$migrate_exit" != "0" ]]; then
|
||||||
|
echo "The clean deployment did not produce the expected topology." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mapfile -t app_containers < <("${compose[@]}" ps --quiet web worker)
|
||||||
|
if [[ "${#app_containers[@]}" -ne 4 ]]; then
|
||||||
|
echo "Expected four running application containers." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
expected_image_id=$(docker image inspect "$image" --format '{{.Id}}')
|
||||||
|
for container in "${app_containers[@]}"; do
|
||||||
|
actual_image_id=$(docker inspect "$container" --format '{{.Image}}')
|
||||||
|
if [[ "$actual_image_id" != "$expected_image_id" ]]; then
|
||||||
|
echo "An application container is not using the archived-source image." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf '%s\n' "$expected_image_id" >"$output_dir/application-image-id.txt"
|
||||||
|
|
||||||
|
expected_migrations=$(
|
||||||
|
find "$workspace/priv/repo/migrations" -type f -name '[0-9]*.exs' |
|
||||||
|
wc -l
|
||||||
|
)
|
||||||
|
"${compose[@]}" exec -T db \
|
||||||
|
psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
||||||
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
|
--command "SELECT version FROM schema_migrations ORDER BY version;" \
|
||||||
|
>"$output_dir/migrations-before-repeat.txt"
|
||||||
|
actual_migrations=$(wc -l <"$output_dir/migrations-before-repeat.txt")
|
||||||
|
|
||||||
|
if [[ "$actual_migrations" -ne "$expected_migrations" ]]; then
|
||||||
|
echo "Fresh migration count is $actual_migrations; expected $expected_migrations." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
category_count_before=$(
|
||||||
|
"${compose[@]}" exec -T db \
|
||||||
|
psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
||||||
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
|
--command "SELECT count(*) FROM categories;"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$category_count_before" -le 0 ]]; then
|
||||||
|
echo "Fresh migration did not seed any product category." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
"${compose[@]}" run --rm migrate >"$output_dir/repeated-migrate.txt"
|
||||||
|
|
||||||
|
"${compose[@]}" exec -T db \
|
||||||
|
psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
||||||
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
|
--command "SELECT version FROM schema_migrations ORDER BY version;" \
|
||||||
|
>"$output_dir/migrations-after-repeat.txt"
|
||||||
|
category_count_after=$(
|
||||||
|
"${compose[@]}" exec -T db \
|
||||||
|
psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
||||||
|
--username "$postgres_user" --dbname "$postgres_db" \
|
||||||
|
--command "SELECT count(*) FROM categories;"
|
||||||
|
)
|
||||||
|
|
||||||
|
if ! diff -u "$output_dir/migrations-before-repeat.txt" \
|
||||||
|
"$output_dir/migrations-after-repeat.txt" >"$output_dir/migrations-repeat.diff" ||
|
||||||
|
[[ "$category_count_before" -ne "$category_count_after" ]]; then
|
||||||
|
echo "The repeated migration changed migration or category counts." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'expected_migrations=%s\n' "$expected_migrations"
|
||||||
|
printf 'actual_migrations=%s\n' "$actual_migrations"
|
||||||
|
printf 'categories_before_repeat=%s\n' "$category_count_before"
|
||||||
|
printf 'categories_after_repeat=%s\n' "$category_count_after"
|
||||||
|
} >"$output_dir/database.txt"
|
||||||
|
|
||||||
|
published=$("${compose[@]}" port proxy 80 | head -n 1)
|
||||||
|
http_port=${published##*:}
|
||||||
|
mailpit_published=$("${compose[@]}" port mailpit 8025 | head -n 1)
|
||||||
|
mailpit_port=${mailpit_published##*:}
|
||||||
|
|
||||||
|
if [[ ! "$http_port" =~ ^[1-9][0-9]*$ ||
|
||||||
|
! "$mailpit_port" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "Compose did not publish dynamic local ports." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
--header 'Host: portability.local' \
|
||||||
|
"http://127.0.0.1:$http_port/healthz/ready" \
|
||||||
|
>"$output_dir/readiness.json"
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
--header 'Host: portability.local' \
|
||||||
|
"http://127.0.0.1:$http_port/" \
|
||||||
|
>"$output_dir/home.html"
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
--header 'Host: portability.local' \
|
||||||
|
"http://127.0.0.1:$http_port/users/register" \
|
||||||
|
>"$output_dir/register.html"
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
"http://127.0.0.1:$mailpit_port/api/v1/info" \
|
||||||
|
>"$output_dir/mailpit.json"
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$workspace"
|
||||||
|
COMPOSE_PROJECT_NAME="$project" \
|
||||||
|
./scripts/verify-realtime-cluster.sh compose
|
||||||
|
) >"$output_dir/pubsub-probe.txt"
|
||||||
|
|
||||||
|
"${compose[@]}" exec -T web /app/bin/who_need_help rpc '
|
||||||
|
nodes = [node() | Node.list()] |> Enum.sort()
|
||||||
|
result = %{node_count: length(nodes), nodes: nodes}
|
||||||
|
IO.inspect(result)
|
||||||
|
if result.node_count != 4, do: exit({:unexpected_cluster_size, result})
|
||||||
|
' >"$output_dir/cluster-size.txt"
|
||||||
|
|
||||||
|
docker inspect "${app_containers[@]}" |
|
||||||
|
jq '[
|
||||||
|
.[] | {
|
||||||
|
name: .Name,
|
||||||
|
image: .Image,
|
||||||
|
restart_count: .RestartCount,
|
||||||
|
oom_killed: .State.OOMKilled,
|
||||||
|
status: .State.Status,
|
||||||
|
health: (.State.Health.Status // null)
|
||||||
|
}
|
||||||
|
]' >"$output_dir/application-containers.json"
|
||||||
|
|
||||||
|
if ! jq -e '
|
||||||
|
length == 4 and
|
||||||
|
all(
|
||||||
|
.status == "running" and
|
||||||
|
.restart_count == 0 and
|
||||||
|
.oom_killed == false and
|
||||||
|
(.health == null or .health == "healthy")
|
||||||
|
)
|
||||||
|
' "$output_dir/application-containers.json" >/dev/null; then
|
||||||
|
echo "A clean-deployment application container is unhealthy." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '%s\n' \
|
||||||
|
"tracked archive clean deployment passed: $project" \
|
||||||
|
"2 web + 2 worker replicas, current migrations, repeated migrate, HTTP, Mailpit, cluster, and PubSub passed"
|
||||||
Loading…
Reference in New Issue
Block a user