430 lines
15 KiB
Bash
Executable File
430 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
action=${1:-}
|
|
root=${2:-/srv/who_need_help-test}
|
|
expected_domain=${3:-test.whoneedhelp.com}
|
|
bundle=${4:-}
|
|
target_commit=${5:-}
|
|
backup=${6:-}
|
|
expected_migration_policy=${7:-}
|
|
image_archive=${8:-}
|
|
image_manifest=${9:-}
|
|
|
|
usage() {
|
|
echo "Usage: $0 plan /srv/who_need_help-test test.whoneedhelp.com" >&2
|
|
echo " $0 apply /srv/who_need_help-test test.whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2
|
|
}
|
|
|
|
case "$action" in
|
|
plan | apply) ;;
|
|
*) usage; exit 2 ;;
|
|
esac
|
|
|
|
for command in curl docker flock git gzip jq pg_restore realpath sha256sum; do
|
|
command -v "$command" >/dev/null 2>&1 || {
|
|
echo "Required test release command is unavailable: $command" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
root=$(realpath --canonicalize-existing "$root")
|
|
[[ "$root" == /srv/who_need_help-test ]] || {
|
|
echo "Refusing a test release outside /srv/who_need_help-test." >&2
|
|
exit 2
|
|
}
|
|
|
|
env_file="$root/.env"
|
|
[[ -f "$env_file" && "$(stat -c '%a' "$env_file")" == 600 ]] || {
|
|
echo "Test .env is missing or does not have mode 0600." >&2
|
|
exit 2
|
|
}
|
|
|
|
read_value() {
|
|
local key=$1
|
|
awk -v key="$key" '
|
|
index($0, key "=") == 1 {
|
|
value = substr($0, length(key) + 2)
|
|
count += 1
|
|
}
|
|
END {
|
|
if (count != 1) {
|
|
printf "Expected exactly one %s entry in the test environment; found %d.\n", key, count > "/dev/stderr"
|
|
exit 1
|
|
}
|
|
print value
|
|
}
|
|
' "$env_file"
|
|
}
|
|
|
|
critical_env_keys=(
|
|
DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY
|
|
PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE
|
|
POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD DATABASE_URL
|
|
)
|
|
for critical_key in "${critical_env_keys[@]}"; do
|
|
read_value "$critical_key" >/dev/null
|
|
done
|
|
|
|
deployment_environment=$(read_value DEPLOYMENT_ENV)
|
|
compose_project=$(read_value COMPOSE_PROJECT_NAME)
|
|
database_mode=$(read_value DATABASE_MODE)
|
|
app_topology=$(read_value APP_TOPOLOGY)
|
|
phx_host=$(read_value PHX_HOST)
|
|
public_origin=$(read_value WNH_BASE_URL)
|
|
current_commit=$(git -C "$root" rev-parse --verify HEAD)
|
|
|
|
[[ "$deployment_environment" == test ]] || {
|
|
echo "DEPLOYMENT_ENV is not test." >&2
|
|
exit 2
|
|
}
|
|
[[ "$compose_project" == who_need_help_test ]] || {
|
|
echo "Unexpected test Compose project." >&2
|
|
exit 2
|
|
}
|
|
[[ "$database_mode" == container ]] || {
|
|
echo "The verified test workflow expects DATABASE_MODE=container." >&2
|
|
exit 2
|
|
}
|
|
[[ "$phx_host" == "$expected_domain" &&
|
|
"$public_origin" == "https://$expected_domain" ]] || {
|
|
echo "Test origin does not match the expected domain." >&2
|
|
exit 2
|
|
}
|
|
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
|
|
echo "Test checkout has uncommitted files." >&2
|
|
exit 2
|
|
}
|
|
|
|
"$root/scripts/compose.sh" "$env_file" config --quiet
|
|
|
|
[[ "$app_topology" == compact ]] || {
|
|
echo "The verified single-server test release workflow requires APP_TOPOLOGY=compact." >&2
|
|
exit 2
|
|
}
|
|
expected_services=(app)
|
|
runtime_services=(app)
|
|
|
|
verify_service_image() {
|
|
local service=$1 expected_image=$2 expected_image_id=$3 require_health=$4
|
|
local container state health configured_image running_image_id
|
|
|
|
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
|
|
[[ ${#containers[@]} -gt 0 ]] || {
|
|
echo "Candidate test service has no container: $service" >&2
|
|
return 1
|
|
}
|
|
|
|
for container in "${containers[@]}"; do
|
|
state=$(docker inspect --format '{{.State.Status}}' "$container")
|
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
|
configured_image=$(docker inspect --format '{{.Config.Image}}' "$container")
|
|
running_image_id=$(docker inspect --format '{{.Image}}' "$container")
|
|
|
|
[[ "$state" == running ]] || {
|
|
echo "Candidate test container is not running: $service" >&2
|
|
return 1
|
|
}
|
|
if [[ "$require_health" == true && "$health" != healthy ]]; then
|
|
echo "Candidate test container is not healthy: $service" >&2
|
|
return 1
|
|
fi
|
|
[[ "$configured_image" == "$expected_image" &&
|
|
"$running_image_id" == "$expected_image_id" ]] || {
|
|
echo "Candidate test service does not use its approved image: $service" >&2
|
|
return 1
|
|
}
|
|
done
|
|
}
|
|
|
|
current_app_image=$(read_value APP_IMAGE)
|
|
current_postgis_image=$(read_value POSTGIS_IMAGE)
|
|
current_app_image_id=$(docker image inspect --format '{{.Id}}' "$current_app_image")
|
|
current_postgis_image_id=$(docker image inspect --format '{{.Id}}' "$current_postgis_image")
|
|
verify_service_image app "$current_app_image" "$current_app_image_id" true
|
|
verify_service_image db "$current_postgis_image" "$current_postgis_image_id" true
|
|
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null
|
|
|
|
printf 'Test checkout: %s\n' "$root"
|
|
printf 'Current commit: %s\n' "$current_commit"
|
|
printf 'Compose project: %s\n' "$compose_project"
|
|
printf 'Topology: %s\n' "$app_topology"
|
|
printf 'Database mode: %s\n' "$database_mode"
|
|
printf 'Public readiness: passed\n'
|
|
df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root"
|
|
|
|
if [[ "$action" == plan ]]; then
|
|
echo "Read-only test release scope check passed."
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
|
|
-z "$expected_migration_policy" || -z "$image_archive" ||
|
|
-z "$image_manifest" ]]; then
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
expected_confirmation="$expected_domain:$target_commit"
|
|
[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$expected_confirmation" ]] || {
|
|
echo "Set WNH_TEST_RELEASE_CONFIRM=$expected_confirmation for the approved test release." >&2
|
|
exit 2
|
|
}
|
|
|
|
for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
|
|
"$image_archive" "$image_archive.sha256" "$image_manifest"; do
|
|
[[ -f "$required_file" ]] || {
|
|
echo "Required test release evidence is missing: $required_file" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
(
|
|
cd "$(dirname -- "$bundle")"
|
|
sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null
|
|
)
|
|
(
|
|
cd "$(dirname -- "$backup")"
|
|
sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
|
|
)
|
|
pg_restore --list "$backup" >/dev/null
|
|
(
|
|
cd "$(dirname -- "$image_archive")"
|
|
sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null
|
|
)
|
|
gzip -t "$image_archive"
|
|
grep -Fx 'format=1' "$image_manifest" >/dev/null
|
|
grep -Fx 'deployment=test' "$image_manifest" >/dev/null
|
|
grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
|
|
grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
|
|
git -C "$root" bundle verify "$bundle" >/dev/null
|
|
|
|
exec {release_lock_fd}>"$root/.git/wnh-test-release.lock"
|
|
chmod 600 "$root/.git/wnh-test-release.lock"
|
|
flock -n "$release_lock_fd" || {
|
|
echo "Another test release already holds $root/.git/wnh-test-release.lock." >&2
|
|
exit 1
|
|
}
|
|
|
|
bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
|
|
[[ "$bundle_head" == "$target_commit" ]] || {
|
|
echo "Bundle HEAD does not match the approved test commit." >&2
|
|
exit 2
|
|
}
|
|
|
|
release_ref="refs/wnh/test-releases/$target_commit"
|
|
git -C "$root" fetch "$bundle" "HEAD:$release_ref"
|
|
[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || {
|
|
echo "Fetched test release ref does not match the approved commit." >&2
|
|
exit 1
|
|
}
|
|
git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || {
|
|
echo "Test updates must be a fast-forward from the deployed commit." >&2
|
|
exit 1
|
|
}
|
|
|
|
policy_script=$(mktemp)
|
|
# Invoked by the EXIT/HUP/INT/TERM traps below.
|
|
# shellcheck disable=SC2329
|
|
cleanup_policy_script() {
|
|
local status=$?
|
|
trap - EXIT HUP INT TERM
|
|
rm -f "$policy_script"
|
|
exit "$status"
|
|
}
|
|
trap cleanup_policy_script EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script"
|
|
chmod 700 "$policy_script"
|
|
migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root")
|
|
rm -f "$policy_script"
|
|
unset -f cleanup_policy_script
|
|
trap - EXIT HUP INT TERM
|
|
printf '%s\n' "$migration_policy_output"
|
|
migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output")
|
|
[[ "$migration_policy" == "$expected_migration_policy" ]] || {
|
|
echo "Remote migration policy does not match the locally approved policy." >&2
|
|
exit 2
|
|
}
|
|
|
|
if [[ "$migration_policy" == forward_only ]]; then
|
|
forward_confirmation="$expected_domain:$target_commit:forward-only"
|
|
[[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
|
|
echo "Set WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation for this test schema boundary." >&2
|
|
exit 2
|
|
}
|
|
fi
|
|
|
|
release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}"
|
|
release_dir="$root/output/releases/$release_id"
|
|
[[ -d "$root/output/releases" ]] || {
|
|
echo "Test release output directory is missing: $root/output/releases" >&2
|
|
exit 2
|
|
}
|
|
install -d -m 700 "$release_dir"
|
|
rollback_manifest="$release_dir/rollback-manifest.txt"
|
|
{
|
|
printf 'previous_commit=%s\n' "$current_commit"
|
|
printf 'target_commit=%s\n' "$target_commit"
|
|
printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)"
|
|
printf 'APP_IMAGE_ID=%s\n' "$current_app_image_id"
|
|
printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)"
|
|
printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)"
|
|
printf 'POSTGIS_IMAGE_ID=%s\n' "$current_postgis_image_id"
|
|
printf 'migration_policy=%s\n' "$migration_policy"
|
|
printf 'database_backup=%s\n' "$backup"
|
|
printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'status=started\n'
|
|
} >"$rollback_manifest"
|
|
chmod 600 "$rollback_manifest"
|
|
|
|
revision_changed=false
|
|
migration_started=false
|
|
|
|
restore_previous_revision() {
|
|
local temporary
|
|
temporary=$(mktemp "$root/.env.test-release-rollback.XXXXXX")
|
|
chmod 600 "$temporary"
|
|
APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
|
SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
|
POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
|
awk '
|
|
BEGIN {
|
|
replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
|
|
replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
|
|
replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
|
|
}
|
|
{
|
|
separator = index($0, "=")
|
|
key = separator > 1 ? substr($0, 1, separator - 1) : ""
|
|
print (key in replacement) ? key "=" replacement[key] : $0
|
|
}
|
|
' "$env_file" >"$temporary"
|
|
mv "$temporary" "$env_file"
|
|
chmod 600 "$env_file"
|
|
git -C "$root" checkout --detach "$current_commit" >/dev/null
|
|
}
|
|
|
|
rollback_runtime() {
|
|
local status=$?
|
|
local rollback_ok=false
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [[ "$status" -ne 0 && "$revision_changed" == true &&
|
|
"$migration_policy" == forward_only && "$migration_started" == true ]]; then
|
|
{
|
|
printf 'status=forward-only-release-failed\n'
|
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'automatic_application_rollback=blocked\n'
|
|
} >>"$rollback_manifest"
|
|
echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2
|
|
elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then
|
|
echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2
|
|
previous_app_image=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest")
|
|
previous_app_image_id=$(awk -F= '$1 == "APP_IMAGE_ID" {print substr($0,index($0,"=")+1)}' "$rollback_manifest")
|
|
previous_postgis_image=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest")
|
|
previous_postgis_image_id=$(awk -F= '$1 == "POSTGIS_IMAGE_ID" {print substr($0,index($0,"=")+1)}' "$rollback_manifest")
|
|
|
|
if restore_previous_revision &&
|
|
"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db &&
|
|
verify_service_image db "$previous_postgis_image" "$previous_postgis_image_id" true &&
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
up -d --no-deps --no-build --force-recreate --wait \
|
|
"${runtime_services[@]}" &&
|
|
verify_service_image app "$previous_app_image" "$previous_app_image_id" true &&
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null; then
|
|
rollback_ok=true
|
|
fi
|
|
|
|
if [[ "$rollback_ok" == true ]]; then
|
|
{
|
|
printf 'status=runtime-rolled-back\n'
|
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
} >>"$rollback_manifest"
|
|
else
|
|
{
|
|
printf 'status=rollback-failed\n'
|
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
} >>"$rollback_manifest"
|
|
echo "Automatic test rollback failed; inspect the rollback manifest and runtime before retrying." >&2
|
|
fi
|
|
fi
|
|
exit "$status"
|
|
}
|
|
trap rollback_runtime EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
gzip -dc "$image_archive" | docker load >/dev/null
|
|
git -C "$root" checkout --detach "$release_ref" >/dev/null
|
|
revision_changed=true
|
|
"$root/scripts/set-deployment-revision.sh" "$env_file"
|
|
"$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain"
|
|
|
|
expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)")
|
|
declare -A approved_image_ids=()
|
|
grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null
|
|
test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}"
|
|
for image in "${expected_images[@]}"; do
|
|
expected_id=$(awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' "$image_manifest")
|
|
[[ -n "$expected_id" ]] || {
|
|
echo "Image manifest is missing the expected image: $image" >&2
|
|
exit 2
|
|
}
|
|
approved_image_ids["$image"]=$expected_id
|
|
[[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || {
|
|
echo "Loaded test image ID does not match the manifest: $image" >&2
|
|
exit 2
|
|
}
|
|
[[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || {
|
|
echo "Loaded test image is not linux/amd64: $image" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
"$root/scripts/restore-drill-compose.sh" "$backup"
|
|
|
|
if [[ "$migration_policy" == forward_only ]]; then
|
|
echo "Stopping the old test application before the forward-only migration boundary."
|
|
"$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}"
|
|
fi
|
|
|
|
"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db
|
|
expected_postgis_image=$(read_value POSTGIS_IMAGE)
|
|
verify_service_image db "$expected_postgis_image" \
|
|
"${approved_image_ids[$expected_postgis_image]}" true
|
|
migration_started=true
|
|
"$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate
|
|
"$root/scripts/check-database.sh" "$env_file" </dev/null
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}"
|
|
|
|
expected_app_image=$(read_value APP_IMAGE)
|
|
for service in "${expected_services[@]}"; do
|
|
verify_service_image "$service" "$expected_app_image" \
|
|
"${approved_image_ids[$expected_app_image]}" true
|
|
done
|
|
|
|
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-realtime-cluster.sh" compose
|
|
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-beam-runtime.sh" compose
|
|
curl --fail --silent --show-error --max-time 30 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null
|
|
|
|
{
|
|
printf 'status=success\n'
|
|
printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
} >>"$rollback_manifest"
|
|
revision_changed=false
|
|
trap - EXIT HUP INT TERM
|
|
|
|
printf 'Test release completed: %s\n' "$target_commit"
|
|
printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
|
|
printf 'Database backup: %s\n' "$backup"
|