diff --git a/scripts/production-release-drill.sh b/scripts/production-release-drill.sh index 69c82a6..2825de1 100755 --- a/scripts/production-release-drill.sh +++ b/scripts/production-release-drill.sh @@ -95,7 +95,7 @@ case "$*" in printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" exit 0 ;; - 'up -d --no-deps --no-build --wait app') + 'up -d --no-deps --no-build --force-recreate --wait app') printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" if [ "${MOCK_FAIL_APP_UP:-}" = once ] && [ ! -e "$MOCK_FAIL_APP_MARKER" ]; then @@ -153,10 +153,24 @@ if [ "$1" = inspect ]; then case "$3" in '{{.State.Status}}') printf 'running\n' ;; '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;; + '{{.Config.Image}}') + if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then + printf 'who-need-help:production-wrong\n' + else + awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \ + "$MOCK_ENV_FILE" + fi + ;; + '{{.Image}}') printf 'sha256:mock-candidate\n' ;; *) exit 1 ;; esac exit 0 fi +if [ "$1" = image ] && [ "$2" = inspect ] && + [ "$3" = --format ] && [ "$4" = '{{.Id}}' ]; then + printf 'sha256:mock-candidate\n' + exit 0 +fi printf 'Unexpected docker invocation: %s\n' "$*" >&2 exit 1 EOF @@ -173,6 +187,7 @@ container_env=( --env "MOCK_TARGET_COMMIT=$target_commit" --env "MOCK_GIT_STATE=$remote_root/git-state" --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" + --env "MOCK_ENV_FILE=$remote_root/.env" --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ) container_mounts=( @@ -215,7 +230,7 @@ grep -F 'compose:build migrate' "$fixture/mock-commands.log" >/dev/null grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null grep -F 'compose:run --rm --no-deps migrate' \ "$fixture/mock-commands.log" >/dev/null -grep -F 'compose:up -d --no-deps --no-build --wait app' \ +grep -F 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log" >/dev/null grep -R -F 'migration_policy=forward_only' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null @@ -231,6 +246,26 @@ if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then exit 1 fi +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +set +e +run_release forward_only \ + --env MOCK_WRONG_RUNTIME_IMAGE=true \ + >"$run_dir/wrong-runtime-image.out" 2>&1 +wrong_runtime_status=$? +set -e +if [[ "$wrong_runtime_status" -eq 0 ]]; then + echo "Release drill accepted a container created from the wrong image." >&2 + exit 1 +fi +grep -F 'Candidate runtime selected an unexpected image' \ + "$run_dir/wrong-runtime-image.out" >/dev/null +grep -F 'previous application will not be restarted' \ + "$run_dir/wrong-runtime-image.out" >/dev/null +grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ + "$fixture/.env" >/dev/null + write_old_env printf '%s\n' "$current_commit" >"$fixture/git-state" : >"$fixture/mock-commands.log" @@ -252,7 +287,7 @@ grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null -test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \ +test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 1 grep -R -F 'status=forward-only-release-failed' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null @@ -278,7 +313,7 @@ grep -Fx "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null -test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \ +test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 2 echo "Isolated production release success/forward-only/safe-recovery drill passed." diff --git a/scripts/production-release-remote.sh b/scripts/production-release-remote.sh index 4508aea..1815fa8 100755 --- a/scripts/production-release-remote.sh +++ b/scripts/production-release-remote.sh @@ -261,7 +261,8 @@ rollback_runtime() { echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2 restore_image_revision "$root/scripts/compose.sh" "$env_file" \ - up -d --no-deps --no-build --wait "${runtime_services[@]}" || true + up -d --no-deps --no-build --force-recreate --wait \ + "${runtime_services[@]}" || true curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null || true @@ -286,6 +287,48 @@ case "$app_topology" in ;; esac +verify_candidate_runtime() { + local expected_app_image expected_app_image_id service container + local configured_image running_image_id state health + + expected_app_image=$(read_value APP_IMAGE) + expected_app_image_id=$( + docker image inspect --format '{{.Id}}' "$expected_app_image" + ) + + for service in "${expected_services[@]}"; do + mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") + [[ ${#containers[@]} -gt 0 ]] || { + echo "Candidate runtime service has no container: $service" >&2 + return 1 + } + + for container in "${containers[@]}"; do + state=$(docker inspect --format '{{.State.Status}}' "$container") + health=$( + docker inspect \ + --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + "$container" + ) + configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") + running_image_id=$(docker inspect --format '{{.Image}}' "$container") + + [[ "$state" == "running" && "$health" == "healthy" ]] || { + echo "Candidate runtime container is not healthy: $service" >&2 + return 1 + } + [[ "$configured_image" == "$expected_app_image" ]] || { + echo "Candidate runtime selected an unexpected image: $service" >&2 + return 1 + } + [[ "$running_image_id" == "$expected_app_image_id" ]] || { + echo "Candidate runtime image ID does not match the selected immutable image: $service" >&2 + return 1 + } + done + done +} + trap rollback_runtime EXIT HUP INT TERM if [[ "$branch" == "main" ]]; then @@ -312,7 +355,8 @@ migration_started=true "$root/scripts/compose.sh" "$env_file" run --rm --no-deps migrate "$root/scripts/check-database.sh" "$env_file" "$root/scripts/compose.sh" "$env_file" \ - up -d --no-deps --no-build --wait "${runtime_services[@]}" + up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}" +verify_candidate_runtime COMPOSE_PROJECT_NAME="$compose_project" \ "$root/scripts/verify-realtime-cluster.sh" compose