283 lines
9.3 KiB
Bash
Executable File
283 lines
9.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
|
BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd"
|
|
mkdir -p "$ROOT/output"
|
|
run_dir=$(mktemp -d "$ROOT/output/rollback-drill.XXXXXX")
|
|
fixture="$run_dir/production"
|
|
mock_bin="$run_dir/mock-bin"
|
|
remote_root=/srv/who_need_help-production
|
|
manifest="$remote_root/output/releases/release-1/rollback-manifest.txt"
|
|
target_commit=2222222222222222222222222222222222222222
|
|
previous_commit=1111111111111111111111111111111111111111
|
|
confirmation="whoneedhelp.com:$target_commit:$previous_commit"
|
|
edge_image=who-need-help:caddy-production-frozen-edge
|
|
|
|
cleanup() {
|
|
trap - EXIT HUP INT TERM
|
|
find "$run_dir" -xdev -depth -delete 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT HUP INT TERM
|
|
|
|
install -d -m 700 \
|
|
"$fixture/.git" \
|
|
"$fixture/scripts" \
|
|
"$fixture/output/releases/release-1" \
|
|
"$fixture/output/backups/production" \
|
|
"$mock_bin"
|
|
|
|
install -m 600 /dev/null "$fixture/.env"
|
|
printf '%s\n' \
|
|
'DEPLOYMENT_ENV=production' \
|
|
'COMPOSE_PROJECT_NAME=who_need_help_production' \
|
|
'DATABASE_MODE=external' \
|
|
'APP_TOPOLOGY=compact' \
|
|
'PHX_HOST=whoneedhelp.com' \
|
|
'WNH_BASE_URL=https://whoneedhelp.com' \
|
|
'EDGE_COMPOSE_PROJECT_NAME=who_need_help_edge' \
|
|
"APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
|
|
"SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}" \
|
|
"POSTGIS_IMAGE=who-need-help:postgis-production-${target_commit:0:12}" \
|
|
"CADDY_IMAGE=$edge_image" \
|
|
>"$fixture/.env"
|
|
|
|
backup="$fixture/output/backups/production/pre-release.dump"
|
|
printf 'isolated rollback drill backup\n' >"$backup"
|
|
backup_hash=$(sha256sum "$backup" | awk '{print $1}')
|
|
printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256"
|
|
printf 'environment=production\n' >"$backup.metadata"
|
|
chmod 600 "$backup" "$backup.sha256" "$backup.metadata"
|
|
|
|
install -m 600 /dev/null "$fixture/output/releases/release-1/rollback-manifest.txt"
|
|
printf '%s\n' \
|
|
"previous_commit=$previous_commit" \
|
|
"target_commit=$target_commit" \
|
|
"APP_IMAGE=who-need-help:production-${previous_commit:0:12}" \
|
|
"SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${previous_commit:0:12}" \
|
|
"POSTGIS_IMAGE=who-need-help:postgis-production-${previous_commit:0:12}" \
|
|
'migration_policy=application_safe' \
|
|
"database_backup=$remote_root/output/backups/production/pre-release.dump" \
|
|
'status=started' \
|
|
'status=success' \
|
|
>"$fixture/output/releases/release-1/rollback-manifest.txt"
|
|
|
|
install -m 755 /dev/null "$fixture/scripts/validate-production-env.sh"
|
|
printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/validate-production-env.sh"
|
|
|
|
install -m 755 /dev/null "$fixture/scripts/compose.sh"
|
|
cat >"$fixture/scripts/compose.sh" <<'EOF'
|
|
#!/bin/sh
|
|
set -eu
|
|
env_file=$1
|
|
shift
|
|
case "$*" in
|
|
'config --quiet') exit 0 ;;
|
|
'ps -q app') printf 'app-1\n'; exit 0 ;;
|
|
up\ *)
|
|
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
|
|
if [ "${MOCK_FAIL_APP_UP:-}" = once ] &&
|
|
[ ! -e "$MOCK_FAIL_APP_MARKER" ]; then
|
|
: >"$MOCK_FAIL_APP_MARKER"
|
|
exit 17
|
|
fi
|
|
exit 0
|
|
;;
|
|
esac
|
|
printf 'Unexpected compose invocation: %s\n' "$*" >&2
|
|
exit 1
|
|
EOF
|
|
|
|
install -m 755 /dev/null "$mock_bin/git"
|
|
cat >"$mock_bin/git" <<'EOF'
|
|
#!/bin/sh
|
|
set -eu
|
|
case " $* " in
|
|
*' status --porcelain --untracked-files=no '*) exit 0 ;;
|
|
*' rev-parse --verify HEAD '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
|
|
*' cat-file -e '*) exit 0 ;;
|
|
*' merge-base --is-ancestor '*) exit 0 ;;
|
|
esac
|
|
printf 'Unexpected git invocation: %s\n' "$*" >&2
|
|
exit 1
|
|
EOF
|
|
|
|
install -m 755 /dev/null "$mock_bin/docker"
|
|
cat >"$mock_bin/docker" <<'EOF'
|
|
#!/bin/sh
|
|
set -eu
|
|
if [ "$1" = image ] && [ "$2" = inspect ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$1" = inspect ]; then
|
|
format=$3
|
|
container=$4
|
|
case "$format" in
|
|
'{{.State.Status}}') printf 'running\n' ;;
|
|
'{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;;
|
|
'{{.Config.Image}}')
|
|
case "$container" in
|
|
app-1) awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' "$MOCK_ENV_FILE" ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
;;
|
|
*) exit 1 ;;
|
|
esac
|
|
exit 0
|
|
fi
|
|
printf 'Unexpected docker invocation: %s\n' "$*" >&2
|
|
exit 1
|
|
EOF
|
|
|
|
for command in curl pg_restore; do
|
|
install -m 755 /dev/null "$mock_bin/$command"
|
|
printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command"
|
|
done
|
|
|
|
touch "$fixture/mock-commands.log"
|
|
chmod 600 "$fixture/mock-commands.log"
|
|
|
|
container_env=(
|
|
--env "MOCK_TARGET_COMMIT=$target_commit"
|
|
--env "MOCK_ENV_FILE=$remote_root/.env"
|
|
--env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log"
|
|
--env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
)
|
|
container_mounts=(
|
|
--volume "$fixture:$remote_root"
|
|
--volume "$mock_bin:/mock-bin:ro"
|
|
--volume "$ROOT/scripts/production-rollback-remote.sh:/runner/production-rollback-remote.sh:ro"
|
|
)
|
|
|
|
docker run --rm \
|
|
--network none \
|
|
--user "$(id -u):$(id -g)" \
|
|
--read-only \
|
|
--tmpfs /tmp:rw,nosuid,nodev,noexec \
|
|
--cap-drop ALL \
|
|
--security-opt no-new-privileges \
|
|
"${container_env[@]}" \
|
|
"${container_mounts[@]}" \
|
|
"$BASE_IMAGE" \
|
|
bash /runner/production-rollback-remote.sh \
|
|
plan "$remote_root" whoneedhelp.com "$manifest" \
|
|
>"$run_dir/plan.out"
|
|
|
|
grep -F "Exact confirmation: $confirmation" "$run_dir/plan.out" >/dev/null
|
|
grep -F 'Read-only production application rollback scope check passed.' \
|
|
"$run_dir/plan.out" >/dev/null
|
|
test ! -s "$fixture/mock-commands.log"
|
|
|
|
docker run --rm \
|
|
--network none \
|
|
--user "$(id -u):$(id -g)" \
|
|
--read-only \
|
|
--tmpfs /tmp:rw,nosuid,nodev,noexec \
|
|
--cap-drop ALL \
|
|
--security-opt no-new-privileges \
|
|
--env "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation" \
|
|
"${container_env[@]}" \
|
|
"${container_mounts[@]}" \
|
|
"$BASE_IMAGE" \
|
|
bash /runner/production-rollback-remote.sh \
|
|
apply "$remote_root" whoneedhelp.com "$manifest" \
|
|
>"$run_dir/apply.out"
|
|
|
|
grep -Fx "APP_IMAGE=who-need-help:production-${previous_commit:0:12}" \
|
|
"$fixture/.env" >/dev/null
|
|
grep -Fx "CADDY_IMAGE=$edge_image" \
|
|
"$fixture/.env" >/dev/null
|
|
grep -F 'compose:up -d --no-deps --no-build --wait app' \
|
|
"$fixture/mock-commands.log" >/dev/null
|
|
if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then
|
|
echo "Application rollback drill touched the shared edge." >&2
|
|
exit 1
|
|
fi
|
|
if grep -E -- 'migrate|--build|(^|[[:space:]])db([[:space:]]|$)' \
|
|
"$fixture/mock-commands.log" >/dev/null; then
|
|
echo "Rollback drill touched migrations, builds, or the database service." >&2
|
|
exit 1
|
|
fi
|
|
find "$fixture/output/releases/release-1" \
|
|
-maxdepth 1 -type f -name 'application-rollback-*.txt' -print -quit |
|
|
grep -q .
|
|
grep -F "Production application images rolled back to release $previous_commit." \
|
|
"$run_dir/apply.out" >/dev/null
|
|
|
|
sed -i \
|
|
's/^migration_policy=application_safe$/migration_policy=forward_only/' \
|
|
"$fixture/output/releases/release-1/rollback-manifest.txt"
|
|
set +e
|
|
docker run --rm \
|
|
--network none \
|
|
--user "$(id -u):$(id -g)" \
|
|
--read-only \
|
|
--tmpfs /tmp:rw,nosuid,nodev,noexec \
|
|
--cap-drop ALL \
|
|
--security-opt no-new-privileges \
|
|
"${container_env[@]}" \
|
|
"${container_mounts[@]}" \
|
|
"$BASE_IMAGE" \
|
|
bash /runner/production-rollback-remote.sh \
|
|
plan "$remote_root" whoneedhelp.com "$manifest" \
|
|
>"$run_dir/forward-only.out" 2>&1
|
|
forward_only_status=$?
|
|
set -e
|
|
if [[ "$forward_only_status" -eq 0 ]]; then
|
|
echo "Rollback drill allowed a forward-only application rollback." >&2
|
|
exit 1
|
|
fi
|
|
grep -F 'automatic old-image rollback is blocked' \
|
|
"$run_dir/forward-only.out" >/dev/null
|
|
sed -i \
|
|
's/^migration_policy=forward_only$/migration_policy=application_safe/' \
|
|
"$fixture/output/releases/release-1/rollback-manifest.txt"
|
|
|
|
sed -i \
|
|
-e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:production-${target_commit:0:12}|" \
|
|
-e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}|" \
|
|
-e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-production-${target_commit:0:12}|" \
|
|
"$fixture/.env"
|
|
: >"$fixture/mock-commands.log"
|
|
rm -f "$fixture/mock-app-failed-once"
|
|
|
|
set +e
|
|
docker run --rm \
|
|
--network none \
|
|
--user "$(id -u):$(id -g)" \
|
|
--read-only \
|
|
--tmpfs /tmp:rw,nosuid,nodev,noexec \
|
|
--cap-drop ALL \
|
|
--security-opt no-new-privileges \
|
|
--env "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation" \
|
|
--env MOCK_FAIL_APP_UP=once \
|
|
--env "MOCK_FAIL_APP_MARKER=$remote_root/mock-app-failed-once" \
|
|
"${container_env[@]}" \
|
|
"${container_mounts[@]}" \
|
|
"$BASE_IMAGE" \
|
|
bash /runner/production-rollback-remote.sh \
|
|
apply "$remote_root" whoneedhelp.com "$manifest" \
|
|
>"$run_dir/failure.out" 2>&1
|
|
failure_status=$?
|
|
set -e
|
|
|
|
if [[ "$failure_status" -eq 0 ]]; then
|
|
echo "Rollback drill did not surface the injected edge failure." >&2
|
|
exit 1
|
|
fi
|
|
grep -F 'Rollback failed; restoring the pre-rollback image selection.' \
|
|
"$run_dir/failure.out" >/dev/null
|
|
grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
|
|
"$fixture/.env" >/dev/null
|
|
grep -Fx "CADDY_IMAGE=$edge_image" \
|
|
"$fixture/.env" >/dev/null
|
|
test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \
|
|
"$fixture/mock-commands.log")" = 2
|
|
if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then
|
|
echo "Rollback recovery touched the shared edge." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Isolated production application rollback plan/apply/failure-recovery drill passed."
|