Verify production runtime image after release

scripts/production-release-drill.sh

@@ -95,7 +95,7 @@ case "$*" in
     printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
     exit 0
     ;;
-  'up -d --no-deps --no-build --wait app')
+  'up -d --no-deps --no-build --force-recreate --wait app')
     printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
     if [ "${MOCK_FAIL_APP_UP:-}" = once ] &&
       [ ! -e "$MOCK_FAIL_APP_MARKER" ]; then
@@ -153,10 +153,24 @@ if [ "$1" = inspect ]; then
   case "$3" in
     '{{.State.Status}}') printf 'running\n' ;;
     '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;;
+    '{{.Config.Image}}')
+      if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then
+        printf 'who-need-help:production-wrong\n'
+      else
+        awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \
+          "$MOCK_ENV_FILE"
+      fi
+      ;;
+    '{{.Image}}') printf 'sha256:mock-candidate\n' ;;
     *) exit 1 ;;
   esac
   exit 0
 fi
+if [ "$1" = image ] && [ "$2" = inspect ] &&
+  [ "$3" = --format ] && [ "$4" = '{{.Id}}' ]; then
+  printf 'sha256:mock-candidate\n'
+  exit 0
+fi
 printf 'Unexpected docker invocation: %s\n' "$*" >&2
 exit 1
 EOF
@@ -173,6 +187,7 @@ container_env=(
   --env "MOCK_TARGET_COMMIT=$target_commit"
   --env "MOCK_GIT_STATE=$remote_root/git-state"
   --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log"
+  --env "MOCK_ENV_FILE=$remote_root/.env"
   --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 )
 container_mounts=(
@@ -215,7 +230,7 @@ grep -F 'compose:build migrate' "$fixture/mock-commands.log" >/dev/null
 grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null
 grep -F 'compose:run --rm --no-deps migrate' \
   "$fixture/mock-commands.log" >/dev/null
-grep -F 'compose:up -d --no-deps --no-build --wait app' \
+grep -F 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
   "$fixture/mock-commands.log" >/dev/null
 grep -R -F 'migration_policy=forward_only' \
   "$fixture/output/releases" --include rollback-manifest.txt >/dev/null
@@ -231,6 +246,26 @@ if grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null; then
   exit 1
 fi
 
+write_old_env
+printf '%s\n' "$current_commit" >"$fixture/git-state"
+: >"$fixture/mock-commands.log"
+set +e
+run_release forward_only \
+  --env MOCK_WRONG_RUNTIME_IMAGE=true \
+  >"$run_dir/wrong-runtime-image.out" 2>&1
+wrong_runtime_status=$?
+set -e
+if [[ "$wrong_runtime_status" -eq 0 ]]; then
+  echo "Release drill accepted a container created from the wrong image." >&2
+  exit 1
+fi
+grep -F 'Candidate runtime selected an unexpected image' \
+  "$run_dir/wrong-runtime-image.out" >/dev/null
+grep -F 'previous application will not be restarted' \
+  "$run_dir/wrong-runtime-image.out" >/dev/null
+grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
+  "$fixture/.env" >/dev/null
+
 write_old_env
 printf '%s\n' "$current_commit" >"$fixture/git-state"
 : >"$fixture/mock-commands.log"
@@ -252,7 +287,7 @@ grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
   "$fixture/.env" >/dev/null
 grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \
   "$fixture/.env" >/dev/null
-test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \
+test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
   "$fixture/mock-commands.log")" = 1
 grep -R -F 'status=forward-only-release-failed' \
   "$fixture/output/releases" --include rollback-manifest.txt >/dev/null
@@ -278,7 +313,7 @@ grep -Fx "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \
   "$fixture/.env" >/dev/null
 grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \
   "$fixture/.env" >/dev/null
-test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \
+test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
   "$fixture/mock-commands.log")" = 2
 
 echo "Isolated production release success/forward-only/safe-recovery drill passed."

scripts/production-release-remote.sh

@@ -261,7 +261,8 @@ rollback_runtime() {
     echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2
     restore_image_revision
     "$root/scripts/compose.sh" "$env_file" \
-      up -d --no-deps --no-build --wait "${runtime_services[@]}" || true
+      up -d --no-deps --no-build --force-recreate --wait \
+      "${runtime_services[@]}" || true
 
     curl --fail --silent --show-error --max-time 15 \
       "https://$expected_domain/healthz/ready" >/dev/null || true
@@ -286,6 +287,48 @@ case "$app_topology" in
     ;;
 esac
 
+verify_candidate_runtime() {
+  local expected_app_image expected_app_image_id service container
+  local configured_image running_image_id state health
+
+  expected_app_image=$(read_value APP_IMAGE)
+  expected_app_image_id=$(
+    docker image inspect --format '{{.Id}}' "$expected_app_image"
+  )
+
+  for service in "${expected_services[@]}"; do
+    mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
+    [[ ${#containers[@]} -gt 0 ]] || {
+      echo "Candidate runtime service has no container: $service" >&2
+      return 1
+    }
+
+    for container in "${containers[@]}"; do
+      state=$(docker inspect --format '{{.State.Status}}' "$container")
+      health=$(
+        docker inspect \
+          --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
+          "$container"
+      )
+      configured_image=$(docker inspect --format '{{.Config.Image}}' "$container")
+      running_image_id=$(docker inspect --format '{{.Image}}' "$container")
+
+      [[ "$state" == "running" && "$health" == "healthy" ]] || {
+        echo "Candidate runtime container is not healthy: $service" >&2
+        return 1
+      }
+      [[ "$configured_image" == "$expected_app_image" ]] || {
+        echo "Candidate runtime selected an unexpected image: $service" >&2
+        return 1
+      }
+      [[ "$running_image_id" == "$expected_app_image_id" ]] || {
+        echo "Candidate runtime image ID does not match the selected immutable image: $service" >&2
+        return 1
+      }
+    done
+  done
+}
+
 trap rollback_runtime EXIT HUP INT TERM
 
 if [[ "$branch" == "main" ]]; then
@@ -312,7 +355,8 @@ migration_started=true
 "$root/scripts/compose.sh" "$env_file" run --rm --no-deps migrate
 "$root/scripts/check-database.sh" "$env_file"
 "$root/scripts/compose.sh" "$env_file" \
-  up -d --no-deps --no-build --wait "${runtime_services[@]}"
+  up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}"
+verify_candidate_runtime
 
 COMPOSE_PROJECT_NAME="$compose_project" \
   "$root/scripts/verify-realtime-cluster.sh" compose