#!/usr/bin/env bash set -euo pipefail umask 077 action=${1:-} root=${2:-/srv/who_need_help-test} expected_domain=${3:-test.whoneedhelp.com} bundle=${4:-} target_commit=${5:-} backup=${6:-} expected_migration_policy=${7:-} image_archive=${8:-} image_manifest=${9:-} usage() { echo "Usage: $0 plan /srv/who_need_help-test test.whoneedhelp.com" >&2 echo " $0 apply /srv/who_need_help-test test.whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2 } case "$action" in plan | apply) ;; *) usage; exit 2 ;; esac for command in curl docker git gzip jq pg_restore realpath sha256sum; do command -v "$command" >/dev/null 2>&1 || { echo "Required test release command is unavailable: $command" >&2 exit 2 } done root=$(realpath --canonicalize-existing "$root") [[ "$root" == /srv/who_need_help-test ]] || { echo "Refusing a test release outside /srv/who_need_help-test." >&2 exit 2 } env_file="$root/.env" [[ -f "$env_file" && "$(stat -c '%a' "$env_file")" == 600 ]] || { echo "Test .env is missing or does not have mode 0600." >&2 exit 2 } read_value() { local key=$1 awk -v key="$key" ' index($0, key "=") == 1 { print substr($0, length(key) + 2) found = 1 exit } END { if (!found) exit 1 } ' "$env_file" } deployment_environment=$(read_value DEPLOYMENT_ENV) compose_project=$(read_value COMPOSE_PROJECT_NAME) database_mode=$(read_value DATABASE_MODE) app_topology=$(read_value APP_TOPOLOGY) phx_host=$(read_value PHX_HOST) public_origin=$(read_value WNH_BASE_URL) current_commit=$(git -C "$root" rev-parse --verify HEAD) [[ "$deployment_environment" == test ]] || { echo "DEPLOYMENT_ENV is not test." >&2 exit 2 } [[ "$compose_project" == who_need_help_test ]] || { echo "Unexpected test Compose project." >&2 exit 2 } [[ "$database_mode" == container ]] || { echo "The verified test workflow expects DATABASE_MODE=container." >&2 exit 2 } [[ "$phx_host" == "$expected_domain" && "$public_origin" == "https://$expected_domain" ]] || { echo "Test origin does not match the expected domain." >&2 exit 2 } [[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || { echo "Test checkout has uncommitted files." >&2 exit 2 } "$root/scripts/compose.sh" "$env_file" config --quiet case "$app_topology" in compact) expected_services=(app) runtime_services=(app) ;; split) expected_services=(web worker) runtime_services=(docker-api-proxy proxy web worker) ;; *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;; esac for service in db "${expected_services[@]}"; do mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") [[ ${#containers[@]} -gt 0 ]] || { echo "Test service is not running: $service" >&2 exit 2 } for container in "${containers[@]}"; do state=$(docker inspect --format '{{.State.Status}}' "$container") health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") [[ "$state" == running && "$health" == healthy ]] || { echo "Test container is not healthy: $service" >&2 exit 2 } done done curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null printf 'Test checkout: %s\n' "$root" printf 'Current commit: %s\n' "$current_commit" printf 'Compose project: %s\n' "$compose_project" printf 'Topology: %s\n' "$app_topology" printf 'Database mode: %s\n' "$database_mode" printf 'Public readiness: passed\n' df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root" if [[ "$action" == plan ]]; then echo "Read-only test release scope check passed." exit 0 fi if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" || -z "$expected_migration_policy" || -z "$image_archive" || -z "$image_manifest" ]]; then usage exit 2 fi expected_confirmation="$expected_domain:$target_commit" [[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$expected_confirmation" ]] || { echo "Set WNH_TEST_RELEASE_CONFIRM=$expected_confirmation for the approved test release." >&2 exit 2 } for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \ "$image_archive" "$image_archive.sha256" "$image_manifest"; do [[ -f "$required_file" ]] || { echo "Required test release evidence is missing: $required_file" >&2 exit 2 } done ( cd "$(dirname -- "$bundle")" sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null ) ( cd "$(dirname -- "$backup")" sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null ) pg_restore --list "$backup" >/dev/null ( cd "$(dirname -- "$image_archive")" sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null ) gzip -t "$image_archive" grep -Fx 'format=1' "$image_manifest" >/dev/null grep -Fx 'deployment=test' "$image_manifest" >/dev/null grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null git -C "$root" bundle verify "$bundle" >/dev/null bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') [[ "$bundle_head" == "$target_commit" ]] || { echo "Bundle HEAD does not match the approved test commit." >&2 exit 2 } release_ref="refs/wnh/test-releases/$target_commit" git -C "$root" fetch "$bundle" "HEAD:$release_ref" [[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || { echo "Fetched test release ref does not match the approved commit." >&2 exit 1 } git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { echo "Test updates must be a fast-forward from the deployed commit." >&2 exit 1 } policy_script=$(mktemp) trap 'rm -f "$policy_script"' EXIT HUP INT TERM git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script" chmod 700 "$policy_script" migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root") rm -f "$policy_script" trap - EXIT HUP INT TERM printf '%s\n' "$migration_policy_output" migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output") [[ "$migration_policy" == "$expected_migration_policy" ]] || { echo "Remote migration policy does not match the locally approved policy." >&2 exit 2 } if [[ "$migration_policy" == forward_only ]]; then forward_confirmation="$expected_domain:$target_commit:forward-only" [[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || { echo "Set WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation for this test schema boundary." >&2 exit 2 } fi release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}" release_dir="$root/output/releases/$release_id" mkdir -p "$release_dir" chmod 700 "$root/output" "$root/output/releases" "$release_dir" rollback_manifest="$release_dir/rollback-manifest.txt" { printf 'previous_commit=%s\n' "$current_commit" printf 'target_commit=%s\n' "$target_commit" printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)" printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)" printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)" printf 'migration_policy=%s\n' "$migration_policy" printf 'database_backup=%s\n' "$backup" printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'status=started\n' } >"$rollback_manifest" chmod 600 "$rollback_manifest" revision_changed=false migration_started=false restore_previous_revision() { local temporary temporary=$(mktemp "$root/.env.test-release-rollback.XXXXXX") chmod 600 "$temporary" APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ awk ' BEGIN { replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"] replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"] replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"] } { separator = index($0, "=") key = separator > 1 ? substr($0, 1, separator - 1) : "" print (key in replacement) ? key "=" replacement[key] : $0 } ' "$env_file" >"$temporary" mv "$temporary" "$env_file" chmod 600 "$env_file" git -C "$root" checkout --detach "$current_commit" >/dev/null } rollback_runtime() { local status=$? trap - EXIT HUP INT TERM if [[ "$status" -ne 0 && "$revision_changed" == true && "$migration_policy" == forward_only && "$migration_started" == true ]]; then { printf 'status=forward-only-release-failed\n' printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'automatic_application_rollback=blocked\n' } >>"$rollback_manifest" echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2 elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2 restore_previous_revision "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db || true "$root/scripts/compose.sh" "$env_file" \ up -d --no-deps --no-build --force-recreate --wait \ "${runtime_services[@]}" || true { printf 'status=runtime-rolled-back\n' printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } >>"$rollback_manifest" fi exit "$status" } trap rollback_runtime EXIT HUP INT TERM gzip -dc "$image_archive" | docker load >/dev/null git -C "$root" checkout --detach "$release_ref" >/dev/null "$root/scripts/set-deployment-revision.sh" "$env_file" revision_changed=true "$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain" expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)") if [[ "$app_topology" == split ]]; then expected_images+=( "$(read_value SOCKET_PROXY_IMAGE)" "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" ) fi grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}" for image in "${expected_images[@]}"; do expected_id=$(awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' "$image_manifest") [[ -n "$expected_id" ]] || { echo "Image manifest is missing the expected image: $image" >&2 exit 2 } [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || { echo "Loaded test image ID does not match the manifest: $image" >&2 exit 2 } [[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || { echo "Loaded test image is not linux/amd64: $image" >&2 exit 2 } done "$root/scripts/restore-drill-compose.sh" "$backup" if [[ "$migration_policy" == forward_only ]]; then echo "Stopping the old test application before the forward-only migration boundary." "$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}" fi "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db verify_service_image() { local service=$1 expected_image=$2 require_health=$3 local expected_image_id container state health configured_image running_image_id expected_image_id=$(docker image inspect --format '{{.Id}}' "$expected_image") mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") [[ ${#containers[@]} -gt 0 ]] || { echo "Candidate test service has no container: $service" >&2 return 1 } for container in "${containers[@]}"; do state=$(docker inspect --format '{{.State.Status}}' "$container") health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") running_image_id=$(docker inspect --format '{{.Image}}' "$container") [[ "$state" == running ]] || { echo "Candidate test container is not running: $service" >&2 return 1 } if [[ "$require_health" == true && "$health" != healthy ]]; then echo "Candidate test container is not healthy: $service" >&2 return 1 fi [[ "$configured_image" == "$expected_image" && "$running_image_id" == "$expected_image_id" ]] || { echo "Candidate test service does not use its approved image: $service" >&2 return 1 } done } verify_service_image db "$(read_value POSTGIS_IMAGE)" true migration_started=true "$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate "$root/scripts/check-database.sh" "$env_file" /dev/null { printf 'status=success\n' printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } >>"$rollback_manifest" revision_changed=false trap - EXIT HUP INT TERM printf 'Test release completed: %s\n' "$target_commit" printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest" printf 'Database backup: %s\n' "$backup"