#!/usr/bin/env bash set -euo pipefail umask 077 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd" mkdir -p "$ROOT/output" run_dir=$(mktemp -d "$ROOT/output/release-drill.XXXXXX") fixture="$run_dir/production" mock_bin="$run_dir/mock-bin" remote_root=/srv/who_need_help-production current_commit=1111111111111111111111111111111111111111 target_commit=2222222222222222222222222222222222222222 release_confirmation="whoneedhelp.com:$target_commit" forward_confirmation="whoneedhelp.com:$target_commit:forward-only" cleanup() { status=$? trap - EXIT HUP INT TERM if [[ "$status" -ne 0 ]]; then for output in "$run_dir"/*.out; do [[ -f "$output" ]] || continue printf '\n--- %s ---\n' "$(basename -- "$output")" >&2 cat "$output" >&2 done fi find "$run_dir" -xdev -depth -delete 2>/dev/null || true exit "$status" } trap cleanup EXIT HUP INT TERM install -d -m 700 \ "$fixture/.git" \ "$fixture/scripts" \ "$fixture/output/releases/incoming" \ "$fixture/output/backups/production" \ "$mock_bin" write_old_env() { install -m 600 /dev/null "$fixture/.env" printf '%s\n' \ 'DEPLOYMENT_ENV=production' \ 'COMPOSE_PROJECT_NAME=who_need_help_production' \ 'DATABASE_MODE=external' \ 'APP_TOPOLOGY=compact' \ 'PHX_HOST=whoneedhelp.com' \ 'WNH_BASE_URL=https://whoneedhelp.com' \ 'EDGE_COMPOSE_PROJECT_NAME=who_need_help_edge' \ "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \ "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${current_commit:0:12}" \ "POSTGIS_IMAGE=who-need-help:postgis-production-${current_commit:0:12}" \ "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ >"$fixture/.env" } write_old_env bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle" printf 'isolated release drill bundle\n' >"$bundle" bundle_hash=$(sha256sum "$bundle" | awk '{print $1}') printf '%s %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256" image_archive="$fixture/output/releases/incoming/who_need_help-$target_commit-images-linux-amd64.tar.gz" printf 'isolated release drill image archive\n' | gzip -n >"$image_archive" image_hash=$(sha256sum "$image_archive" | awk '{print $1}') printf '%s %s\n' "$image_hash" "$(basename -- "$image_archive")" \ >"$image_archive.sha256" fixture_image_config="$run_dir/fixture-image-config.json" printf '%s\n' \ '{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}' \ >"$fixture_image_config" fixture_image_id="sha256:$(sha256sum "$fixture_image_config" | awk '{print $1}')" [[ "$fixture_image_id" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "Release drill generated an invalid fixture image ID." >&2 exit 1 } image_manifest="$fixture/output/releases/incoming/who_need_help-$target_commit-images.manifest" printf '%s\n' \ 'format=1' \ "commit=$target_commit" \ 'platform=linux/amd64' \ 'topology=compact' \ 'image_count=1' \ "image=who-need-help:production-${target_commit:0:12}|$fixture_image_id" \ >"$image_manifest" backup="$fixture/output/backups/production/pre-release.dump" printf 'isolated release drill backup\n' >"$backup" backup_hash=$(sha256sum "$backup" | awk '{print $1}') printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256" printf 'environment=production\n' >"$backup.metadata" chmod 600 "$bundle" "$bundle.sha256" "$image_archive" \ "$image_archive.sha256" "$image_manifest" "$backup" "$backup.sha256" \ "$backup.metadata" for script in validate-production-env.sh check-environment-readiness.sh \ verify-realtime-cluster.sh verify-beam-runtime.sh; do install -m 755 /dev/null "$fixture/scripts/$script" printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/$script" done install -m 755 "$ROOT/scripts/check-database.sh" \ "$fixture/scripts/check-database.sh" install -m 755 /dev/null "$fixture/scripts/set-deployment-revision.sh" cat >"$fixture/scripts/set-deployment-revision.sh" <<'EOF' #!/bin/sh set -eu env_file=$1 sed -i \ -e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ -e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ -e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ "$env_file" EOF install -m 755 /dev/null "$fixture/scripts/compose.sh" cat >"$fixture/scripts/compose.sh" <<'EOF' #!/bin/sh set -eu env_file=$1 shift case "$*" in 'config --quiet') exit 0 ;; 'ps -q app') printf 'app-1\n'; exit 0 ;; 'stop app') printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" exit 0 ;; 'run --rm --no-deps --interactive=false migrate') printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" exit 0 ;; 'run --rm --no-deps --interactive=false migrate /app/bin/who_need_help eval '*) printf 'compose:check-database-noninteractive\n' >>"$MOCK_COMMAND_LOG" exit 0 ;; 'run --rm --no-deps migrate /app/bin/who_need_help eval '*) cat >/dev/null printf 'compose:check-database-consumed-stdin\n' >>"$MOCK_COMMAND_LOG" exit 0 ;; 'up -d --no-deps --no-build --force-recreate --wait app') printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" if [ "${MOCK_FAIL_APP_UP:-}" = once ] && [ ! -e "$MOCK_FAIL_APP_MARKER" ]; then : >"$MOCK_FAIL_APP_MARKER" exit 23 fi exit 0 ;; esac printf 'Unexpected compose invocation: %s\n' "$*" >&2 exit 1 EOF printf '%s\n' "$current_commit" >"$fixture/git-state" install -m 755 /dev/null "$mock_bin/git" cat >"$mock_bin/git" <<'EOF' #!/bin/sh set -eu case " $* " in *' symbolic-ref --quiet --short HEAD '*) exit 1 ;; *' status --porcelain --untracked-files=normal '*) exit 0 ;; *' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;; *' rev-parse refs/wnh/releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; *' bundle verify '*) exit 0 ;; *' bundle list-heads '*) printf '%s HEAD\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; *' fetch '*) printf 'git:%s\n' "$*" >>"$MOCK_COMMAND_LOG" exit 0 ;; *' merge-base --is-ancestor '*) exit 0 ;; *' show refs/wnh/releases/'*':scripts/release-migration-policy.sh '*) cat <<'POLICY' #!/bin/sh set -eu printf 'migration_policy=%s\n' "$MOCK_MIGRATION_POLICY" printf 'migration_versions=20260101000000:%s\n' "$MOCK_MIGRATION_POLICY" printf 'migration_count=1\n' POLICY exit 0 ;; *' checkout --detach refs/wnh/releases/'*) printf '%s\n' "$MOCK_TARGET_COMMIT" >"$MOCK_GIT_STATE" exit 0 ;; esac printf 'Unexpected git invocation: %s\n' "$*" >&2 exit 1 EOF install -m 755 /dev/null "$mock_bin/docker" cat >"$mock_bin/docker" <<'EOF' #!/bin/sh set -eu if [ "$1" = inspect ]; then case "$3" in '{{.State.Status}}') printf 'running\n' ;; '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;; '{{.Config.Image}}') if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then printf 'who-need-help:production-wrong\n' else awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \ "$MOCK_ENV_FILE" fi ;; '{{.Image}}') printf '%s\n' "$FIXTURE_IMAGE_ID" ;; *) exit 1 ;; esac exit 0 fi if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = --format ]; then case "$4" in '{{.Id}}') printf '%s\n' "$FIXTURE_IMAGE_ID" ;; '{{.Os}}/{{.Architecture}}') printf 'linux/amd64\n' ;; *) exit 1 ;; esac exit 0 fi if [ "$1" = load ]; then cat >/dev/null printf 'docker:load\n' >>"$MOCK_COMMAND_LOG" exit 0 fi printf 'Unexpected docker invocation: %s\n' "$*" >&2 exit 1 EOF for command in curl jq pg_restore; do install -m 755 /dev/null "$mock_bin/$command" printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" done touch "$fixture/mock-commands.log" chmod 600 "$fixture/mock-commands.log" "$fixture/git-state" container_env=( --env "MOCK_TARGET_COMMIT=$target_commit" --env "MOCK_GIT_STATE=$remote_root/git-state" --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" --env "MOCK_ENV_FILE=$remote_root/.env" --env "FIXTURE_IMAGE_ID=$fixture_image_id" --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ) container_mounts=( --volume "$fixture:$remote_root" --volume "$mock_bin:/mock-bin:ro" --volume "$ROOT/scripts/production-release-remote.sh:/runner/production-release-remote.sh:ro" ) run_release() { local policy=$1 shift docker run --rm \ --network none \ --user "$(id -u):$(id -g)" \ --read-only \ --tmpfs /tmp:rw,nosuid,nodev,noexec \ --cap-drop ALL \ --security-opt no-new-privileges \ --env "MOCK_MIGRATION_POLICY=$policy" \ --env "WNH_PRODUCTION_RELEASE_CONFIRM=$release_confirmation" \ --env "WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation" \ "$@" \ "${container_env[@]}" \ "${container_mounts[@]}" \ "$BASE_IMAGE" \ bash -c 'bash -s -- "$@" < /runner/production-release-remote.sh' _ \ apply "$remote_root" whoneedhelp.com \ "$remote_root/output/releases/incoming/$(basename -- "$bundle")" \ "$target_commit" \ "$remote_root/output/backups/production/$(basename -- "$backup")" \ "$policy" \ "$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \ "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")" } run_release forward_only >"$run_dir/forward-success.out" grep -Fx \ "image=who-need-help:production-${target_commit:0:12}|$fixture_image_id" \ "$image_manifest" >/dev/null grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx 'docker:load' "$fixture/mock-commands.log" >/dev/null if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then echo "Production release drill unexpectedly compiled on the production host." >&2 exit 1 fi grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null grep -F 'compose:run --rm --no-deps --interactive=false migrate' \ "$fixture/mock-commands.log" >/dev/null grep -Fx 'compose:check-database-noninteractive' \ "$fixture/mock-commands.log" >/dev/null if grep -F 'compose:check-database-consumed-stdin' \ "$fixture/mock-commands.log" >/dev/null; then echo "Database check consumed the streamed release script." >&2 exit 1 fi grep -F 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log" >/dev/null grep -R -F 'migration_policy=forward_only' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null grep -R -F 'status=success' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null if grep -R -E '^CADDY_IMAGE=' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null; then echo "Application release manifest unexpectedly captured the shared edge image." >&2 exit 1 fi if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then echo "Application release drill touched the shared edge." >&2 exit 1 fi write_old_env printf '%s\n' "$current_commit" >"$fixture/git-state" : >"$fixture/mock-commands.log" set +e run_release forward_only \ --env MOCK_WRONG_RUNTIME_IMAGE=true \ >"$run_dir/wrong-runtime-image.out" 2>&1 wrong_runtime_status=$? set -e if [[ "$wrong_runtime_status" -eq 0 ]]; then echo "Release drill accepted a container created from the wrong image." >&2 exit 1 fi grep -F 'Candidate runtime selected an unexpected image' \ "$run_dir/wrong-runtime-image.out" >/dev/null grep -F 'previous application will not be restarted' \ "$run_dir/wrong-runtime-image.out" >/dev/null grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "$fixture/.env" >/dev/null write_old_env printf '%s\n' "$current_commit" >"$fixture/git-state" : >"$fixture/mock-commands.log" rm -f "$fixture/app-up-failed" set +e run_release forward_only \ --env MOCK_FAIL_APP_UP=once \ --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ >"$run_dir/forward-failure.out" 2>&1 forward_failure_status=$? set -e if [[ "$forward_failure_status" -eq 0 ]]; then echo "Forward-only release drill did not surface the injected startup failure." >&2 exit 1 fi grep -F 'previous application will not be restarted' \ "$run_dir/forward-failure.out" >/dev/null grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 1 grep -R -F 'status=forward-only-release-failed' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null write_old_env printf '%s\n' "$current_commit" >"$fixture/git-state" : >"$fixture/mock-commands.log" rm -f "$fixture/app-up-failed" set +e run_release application_safe \ --env MOCK_FAIL_APP_UP=once \ --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ >"$run_dir/safe-failure.out" 2>&1 safe_failure_status=$? set -e if [[ "$safe_failure_status" -eq 0 ]]; then echo "Application-safe release drill did not surface the injected failure." >&2 exit 1 fi grep -F 'restoring the previous immutable image tags' \ "$run_dir/safe-failure.out" >/dev/null grep -Fx "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 2 echo "Isolated production release success/forward-only/safe-recovery drill passed."