#!/usr/bin/env bash set -euo pipefail umask 077 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd" mkdir -p "$ROOT/output" run_dir=$(mktemp -d "$ROOT/output/rollback-drill.XXXXXX") fixture="$run_dir/production" mock_bin="$run_dir/mock-bin" remote_root=/srv/who_need_help-production manifest="$remote_root/output/releases/release-1/rollback-manifest.txt" target_commit=2222222222222222222222222222222222222222 previous_commit=1111111111111111111111111111111111111111 confirmation="whoneedhelp.com:$target_commit:$previous_commit" edge_image=who-need-help:caddy-production-frozen-edge cleanup() { trap - EXIT HUP INT TERM find "$run_dir" -xdev -depth -delete 2>/dev/null || true } trap cleanup EXIT HUP INT TERM install -d -m 700 \ "$fixture/.git" \ "$fixture/scripts" \ "$fixture/output/releases/release-1" \ "$fixture/output/backups/production" \ "$mock_bin" install -m 600 /dev/null "$fixture/.env" printf '%s\n' \ 'DEPLOYMENT_ENV=production' \ 'COMPOSE_PROJECT_NAME=who_need_help_production' \ 'DATABASE_MODE=external' \ 'APP_TOPOLOGY=compact' \ 'PHX_HOST=whoneedhelp.com' \ 'WNH_BASE_URL=https://whoneedhelp.com' \ 'EDGE_COMPOSE_PROJECT_NAME=who_need_help_edge' \ "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}" \ "POSTGIS_IMAGE=who-need-help:postgis-production-${target_commit:0:12}" \ "CADDY_IMAGE=$edge_image" \ >"$fixture/.env" backup="$fixture/output/backups/production/pre-release.dump" printf 'isolated rollback drill backup\n' >"$backup" backup_hash=$(sha256sum "$backup" | awk '{print $1}') printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256" printf 'environment=production\n' >"$backup.metadata" chmod 600 "$backup" "$backup.sha256" "$backup.metadata" install -m 600 /dev/null "$fixture/output/releases/release-1/rollback-manifest.txt" printf '%s\n' \ "previous_commit=$previous_commit" \ "target_commit=$target_commit" \ "APP_IMAGE=who-need-help:production-${previous_commit:0:12}" \ "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${previous_commit:0:12}" \ "POSTGIS_IMAGE=who-need-help:postgis-production-${previous_commit:0:12}" \ 'migration_policy=application_safe' \ "database_backup=$remote_root/output/backups/production/pre-release.dump" \ 'status=started' \ 'status=success' \ >"$fixture/output/releases/release-1/rollback-manifest.txt" install -m 755 /dev/null "$fixture/scripts/validate-production-env.sh" printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/validate-production-env.sh" install -m 755 /dev/null "$fixture/scripts/compose.sh" cat >"$fixture/scripts/compose.sh" <<'EOF' #!/bin/sh set -eu env_file=$1 shift case "$*" in 'config --quiet') exit 0 ;; 'ps -q app') printf 'app-1\n'; exit 0 ;; up\ *) printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" if [ "${MOCK_FAIL_APP_UP:-}" = once ] && [ ! -e "$MOCK_FAIL_APP_MARKER" ]; then : >"$MOCK_FAIL_APP_MARKER" exit 17 fi exit 0 ;; esac printf 'Unexpected compose invocation: %s\n' "$*" >&2 exit 1 EOF install -m 755 /dev/null "$mock_bin/git" cat >"$mock_bin/git" <<'EOF' #!/bin/sh set -eu case " $* " in *' status --porcelain --untracked-files=no '*) exit 0 ;; *' rev-parse --verify HEAD '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; *' cat-file -e '*) exit 0 ;; *' merge-base --is-ancestor '*) exit 0 ;; esac printf 'Unexpected git invocation: %s\n' "$*" >&2 exit 1 EOF install -m 755 /dev/null "$mock_bin/docker" cat >"$mock_bin/docker" <<'EOF' #!/bin/sh set -eu if [ "$1" = image ] && [ "$2" = inspect ]; then exit 0 fi if [ "$1" = inspect ]; then format=$3 container=$4 case "$format" in '{{.State.Status}}') printf 'running\n' ;; '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;; '{{.Config.Image}}') case "$container" in app-1) awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' "$MOCK_ENV_FILE" ;; *) exit 1 ;; esac ;; *) exit 1 ;; esac exit 0 fi printf 'Unexpected docker invocation: %s\n' "$*" >&2 exit 1 EOF for command in curl pg_restore; do install -m 755 /dev/null "$mock_bin/$command" printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" done touch "$fixture/mock-commands.log" chmod 600 "$fixture/mock-commands.log" container_env=( --env "MOCK_TARGET_COMMIT=$target_commit" --env "MOCK_ENV_FILE=$remote_root/.env" --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ) container_mounts=( --volume "$fixture:$remote_root" --volume "$mock_bin:/mock-bin:ro" --volume "$ROOT/scripts/production-rollback-remote.sh:/runner/production-rollback-remote.sh:ro" ) docker run --rm \ --network none \ --user "$(id -u):$(id -g)" \ --read-only \ --tmpfs /tmp:rw,nosuid,nodev,noexec \ --cap-drop ALL \ --security-opt no-new-privileges \ "${container_env[@]}" \ "${container_mounts[@]}" \ "$BASE_IMAGE" \ bash /runner/production-rollback-remote.sh \ plan "$remote_root" whoneedhelp.com "$manifest" \ >"$run_dir/plan.out" grep -F "Exact confirmation: $confirmation" "$run_dir/plan.out" >/dev/null grep -F 'Read-only production application rollback scope check passed.' \ "$run_dir/plan.out" >/dev/null test ! -s "$fixture/mock-commands.log" docker run --rm \ --network none \ --user "$(id -u):$(id -g)" \ --read-only \ --tmpfs /tmp:rw,nosuid,nodev,noexec \ --cap-drop ALL \ --security-opt no-new-privileges \ --env "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation" \ "${container_env[@]}" \ "${container_mounts[@]}" \ "$BASE_IMAGE" \ bash /runner/production-rollback-remote.sh \ apply "$remote_root" whoneedhelp.com "$manifest" \ >"$run_dir/apply.out" grep -Fx "APP_IMAGE=who-need-help:production-${previous_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=$edge_image" \ "$fixture/.env" >/dev/null grep -F 'compose:up -d --no-deps --no-build --wait app' \ "$fixture/mock-commands.log" >/dev/null if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then echo "Application rollback drill touched the shared edge." >&2 exit 1 fi if grep -E -- 'migrate|--build|(^|[[:space:]])db([[:space:]]|$)' \ "$fixture/mock-commands.log" >/dev/null; then echo "Rollback drill touched migrations, builds, or the database service." >&2 exit 1 fi find "$fixture/output/releases/release-1" \ -maxdepth 1 -type f -name 'application-rollback-*.txt' -print -quit | grep -q . grep -F "Production application images rolled back to release $previous_commit." \ "$run_dir/apply.out" >/dev/null sed -i \ 's/^migration_policy=application_safe$/migration_policy=forward_only/' \ "$fixture/output/releases/release-1/rollback-manifest.txt" set +e docker run --rm \ --network none \ --user "$(id -u):$(id -g)" \ --read-only \ --tmpfs /tmp:rw,nosuid,nodev,noexec \ --cap-drop ALL \ --security-opt no-new-privileges \ "${container_env[@]}" \ "${container_mounts[@]}" \ "$BASE_IMAGE" \ bash /runner/production-rollback-remote.sh \ plan "$remote_root" whoneedhelp.com "$manifest" \ >"$run_dir/forward-only.out" 2>&1 forward_only_status=$? set -e if [[ "$forward_only_status" -eq 0 ]]; then echo "Rollback drill allowed a forward-only application rollback." >&2 exit 1 fi grep -F 'automatic old-image rollback is blocked' \ "$run_dir/forward-only.out" >/dev/null sed -i \ 's/^migration_policy=forward_only$/migration_policy=application_safe/' \ "$fixture/output/releases/release-1/rollback-manifest.txt" sed -i \ -e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:production-${target_commit:0:12}|" \ -e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}|" \ -e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-production-${target_commit:0:12}|" \ "$fixture/.env" : >"$fixture/mock-commands.log" rm -f "$fixture/mock-app-failed-once" set +e docker run --rm \ --network none \ --user "$(id -u):$(id -g)" \ --read-only \ --tmpfs /tmp:rw,nosuid,nodev,noexec \ --cap-drop ALL \ --security-opt no-new-privileges \ --env "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation" \ --env MOCK_FAIL_APP_UP=once \ --env "MOCK_FAIL_APP_MARKER=$remote_root/mock-app-failed-once" \ "${container_env[@]}" \ "${container_mounts[@]}" \ "$BASE_IMAGE" \ bash /runner/production-rollback-remote.sh \ apply "$remote_root" whoneedhelp.com "$manifest" \ >"$run_dir/failure.out" 2>&1 failure_status=$? set -e if [[ "$failure_status" -eq 0 ]]; then echo "Rollback drill did not surface the injected edge failure." >&2 exit 1 fi grep -F 'Rollback failed; restoring the pre-rollback image selection.' \ "$run_dir/failure.out" >/dev/null grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=$edge_image" \ "$fixture/.env" >/dev/null test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \ "$fixture/mock-commands.log")" = 2 if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then echo "Rollback recovery touched the shared edge." >&2 exit 1 fi echo "Isolated production application rollback plan/apply/failure-recovery drill passed."