Build production images off host

2026-08-03 17:03:00 +03:00 · 2026-08-03 17:03:00 +03:00 · b80bf6e9a5
commit b80bf6e9a5
parent 0aac86bda0
5 changed files with 325 additions and 39 deletions
--- a/docs/operations.md
+++ b/docs/operations.md
@ -1077,17 +1077,24 @@ WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \
 The apply path refuses tracked local or remote modifications. It then:

 1. creates and verifies a full Git bundle for exactly that clean commit;
-2. uploads only that bundle to the production checkout's ignored
+2. reads the production environment over SSH into a mode-0600 temporary local
+   file, selects the candidate's immutable image tags, builds the required
+   `linux/amd64` images on the development workstation, and deletes that
+   temporary environment file;
+3. records every image ID in a manifest, creates a timestamp-free
+   gzip-compressed Docker archive, verifies its SHA-256, and uploads the
+   bundle, archive, checksum, and manifest to the production checkout's ignored
   `output/releases/`;
-3. creates a custom-format PostgreSQL 18 backup without exposing the database
+4. creates a custom-format PostgreSQL 18 backup without exposing the database
   password in process arguments;
-4. verifies its archive catalog and SHA-256, then copies and verifies the
+5. verifies its archive catalog and SHA-256, then copies and verifies the
   backup again under local ignored `output/production-backups/`;
-5. fast-forwards the production checkout without accessing or changing the
+6. fast-forwards the production checkout without accessing or changing the
   test checkout or public remote;
-6. selects immutable per-commit application image tags, applies migrations,
-   starts only the application topology, and verifies public readiness through
-   the already-running shared edge plus the Android App Links endpoints.
+7. verifies the transferred archive and manifest, loads the ready images
+   without compiling on the production host, applies migrations, starts only
+   the application topology, and verifies public readiness through the
+   already-running shared edge plus the Android App Links endpoints.

 Every newly added migration must have one reviewed entry in
 `priv/repo/migration_application_compatibility.tsv`. `application_safe` means
@ -1101,12 +1108,13 @@ WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=whoneedhelp.com:FULL_COMMIT:forward-only \
  ./scripts/production-release.sh apply whoneedhelp
 ```

-For a forward-only release, all candidate images are built first, the old
-application is stopped before migration begins, and the target application is
-started only after the migration runner succeeds. If anything fails after the
-migration begins, the release deliberately leaves the old application stopped
-and records that boundary in the release manifest. Restarting an older image
-against a potentially incompatible schema is never automatic.
+For a forward-only release, all candidate images are built and transferred
+first, the old application is stopped before migration begins, and the target
+application is started only after the migration runner succeeds. If anything
+fails after the migration begins, the release deliberately leaves the old
+application stopped and records that boundary in the release manifest.
+Restarting an older image against a potentially incompatible schema is never
+automatic.

 For an `application_safe` release, an application startup failure restores the
 previous immutable application image tags and attempts to recover public
--- a/scripts/prepare-production-images.sh
+++ b/scripts/prepare-production-images.sh
@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+source_env=${1:-}
+
+if [[ -z "$source_env" || ! -f "$source_env" ]]; then
+  echo "Usage: $0 PRODUCTION_ENV_FILE" >&2
+  exit 2
+fi
+
+for command in docker gzip jq sha256sum; do
+  command -v "$command" >/dev/null 2>&1 || {
+    echo "Required command is unavailable: $command" >&2
+    exit 2
+  }
+done
+
+if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
+  echo "Refusing to build production images from a dirty tracked checkout." >&2
+  exit 2
+fi
+
+commit=$(git -C "$ROOT" rev-parse --verify HEAD)
+short_commit=${commit:0:12}
+release_dir="$ROOT/output/releases/$commit"
+archive="$release_dir/who_need_help-$commit-images-linux-amd64.tar.gz"
+checksum="$archive.sha256"
+manifest="$release_dir/who_need_help-$commit-images.manifest"
+
+mkdir -p "$release_dir"
+chmod 700 "$ROOT/output" "$ROOT/output/releases" "$release_dir"
+
+build_env=$(mktemp "$release_dir/.production-image-build.XXXXXX")
+cleanup() {
+  trap - EXIT HUP INT TERM
+  rm -f "$build_env"
+}
+trap cleanup EXIT HUP INT TERM
+install -m 600 "$source_env" "$build_env"
+
+read_value() {
+  local key=$1
+  awk -v key="$key" '
+    index($0, key "=") == 1 {
+      print substr($0, length(key) + 2)
+      found = 1
+      exit
+    }
+    END { if (!found) exit 1 }
+  ' "$build_env"
+}
+
+replace_value() {
+  local key=$1
+  local value=$2
+  local temporary
+  temporary=$(mktemp "$release_dir/.production-image-env.XXXXXX")
+  chmod 600 "$temporary"
+  awk -v key="$key" -v value="$value" '
+    index($0, key "=") == 1 { print key "=" value; found = 1; next }
+    { print }
+    END { if (!found) exit 1 }
+  ' "$build_env" >"$temporary"
+  mv "$temporary" "$build_env"
+  chmod 600 "$build_env"
+}
+
+[[ "$(read_value DEPLOYMENT_ENV)" == production ]] || {
+  echo "The image build input is not a production environment." >&2
+  exit 2
+}
+[[ "$(read_value DATABASE_MODE)" == external ]] || {
+  echo "The verified production image workflow expects DATABASE_MODE=external." >&2
+  exit 2
+}
+
+replace_value APP_IMAGE "who-need-help:production-$short_commit"
+replace_value SOCKET_PROXY_IMAGE \
+  "who-need-help:socket-proxy-production-$short_commit"
+replace_value POSTGIS_IMAGE "who-need-help:postgis-production-$short_commit"
+
+topology=$(read_value APP_TOPOLOGY)
+case "$topology" in
+  compact)
+    build_services=(migrate)
+    ;;
+  split)
+    build_services=(docker-api-proxy proxy migrate)
+    ;;
+  *)
+    echo "APP_TOPOLOGY must be compact or split." >&2
+    exit 2
+    ;;
+esac
+
+app_image=$(read_value APP_IMAGE)
+images=("$app_image")
+if [[ "$topology" == split ]]; then
+  socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE)
+  proxy_image=$(
+    "$ROOT/scripts/compose.sh" "$build_env" config --format json |
+      jq -er '.services.proxy.image'
+  )
+  images+=("$socket_proxy_image" "$proxy_image")
+fi
+
+if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then
+  [[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || {
+    echo "The production image package is incomplete; refusing to overwrite it." >&2
+    exit 2
+  }
+  (
+    cd "$release_dir"
+    sha256sum --check "$(basename -- "$checksum")" >/dev/null
+  )
+  grep -Fx "commit=$commit" "$manifest" >/dev/null
+  echo "Production image package already exists and passed checksum verification."
+else
+  "$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}"
+
+  manifest_tmp=$(mktemp "$release_dir/.production-images-manifest.XXXXXX")
+  archive_tmp=$(mktemp "$release_dir/.production-images-archive.XXXXXX")
+  trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM
+
+  {
+    printf 'format=1\n'
+    printf 'commit=%s\n' "$commit"
+    printf 'platform=linux/amd64\n'
+    printf 'topology=%s\n' "$topology"
+    printf 'image_count=%s\n' "${#images[@]}"
+    for image in "${images[@]}"; do
+      platform=$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")
+      [[ "$platform" == linux/amd64 ]] || {
+        echo "Production image has an unexpected platform: $image ($platform)" >&2
+        exit 2
+      }
+      image_id=$(docker image inspect --format '{{.Id}}' "$image")
+      printf 'image=%s|%s\n' "$image" "$image_id"
+    done
+  } >"$manifest_tmp"
+
+  docker save "${images[@]}" | gzip -n -9 >"$archive_tmp"
+  mv "$archive_tmp" "$archive"
+  mv "$manifest_tmp" "$manifest"
+  chmod 600 "$archive" "$manifest"
+  hash=$(sha256sum "$archive" | awk '{print $1}')
+  printf '%s  %s\n' "$hash" "$(basename -- "$archive")" >"$checksum"
+  chmod 600 "$checksum"
+fi
+
+(
+  cd "$release_dir"
+  sha256sum --check "$(basename -- "$checksum")" >/dev/null
+)
+gzip -t "$archive"
+grep -Fx 'platform=linux/amd64' "$manifest" >/dev/null
+grep -Fx "topology=$topology" "$manifest" >/dev/null
+test "$(grep -c '^image=' "$manifest")" = "${#images[@]}"
+for image in "${images[@]}"; do
+  awk -F'|' -v image="$image" '
+    $1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 }
+    END { if (!found) exit 1 }
+  ' "$manifest"
+done
+
+cleanup
+printf 'Production image archive: %s\n' "$archive"
+printf 'Image archive checksum: %s\n' "$checksum"
+printf 'Image manifest: %s\n' "$manifest"
--- a/scripts/production-release-drill.sh
+++ b/scripts/production-release-drill.sh
@ -13,10 +13,20 @@ current_commit=1111111111111111111111111111111111111111
 target_commit=2222222222222222222222222222222222222222
 release_confirmation="whoneedhelp.com:$target_commit"
 forward_confirmation="whoneedhelp.com:$target_commit:forward-only"
+mock_candidate_id="sha256:$(printf 'who-need-help-release-drill-candidate' | sha256sum | awk '{print $1}')"

 cleanup() {
+  status=$?
  trap - EXIT HUP INT TERM
+  if [[ "$status" -ne 0 ]]; then
+    for output in "$run_dir"/*.out; do
+      [[ -f "$output" ]] || continue
+      printf '\n--- %s ---\n' "$(basename -- "$output")" >&2
+      cat "$output" >&2
+    done
+  fi
  find "$run_dir" -xdev -depth -delete 2>/dev/null || true
+  exit "$status"
 }
 trap cleanup EXIT HUP INT TERM

@ -49,12 +59,28 @@ bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle"
 printf 'isolated release drill bundle\n' >"$bundle"
 bundle_hash=$(sha256sum "$bundle" | awk '{print $1}')
 printf '%s  %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256"
+image_archive="$fixture/output/releases/incoming/who_need_help-$target_commit-images-linux-amd64.tar.gz"
+printf 'isolated release drill image archive\n' | gzip -n >"$image_archive"
+image_hash=$(sha256sum "$image_archive" | awk '{print $1}')
+printf '%s  %s\n' "$image_hash" "$(basename -- "$image_archive")" \
+  >"$image_archive.sha256"
+image_manifest="$fixture/output/releases/incoming/who_need_help-$target_commit-images.manifest"
+printf '%s\n' \
+  'format=1' \
+  "commit=$target_commit" \
+  'platform=linux/amd64' \
+  'topology=compact' \
+  'image_count=1' \
+  "image=who-need-help:production-${target_commit:0:12}|$mock_candidate_id" \
+  >"$image_manifest"
 backup="$fixture/output/backups/production/pre-release.dump"
 printf 'isolated release drill backup\n' >"$backup"
 backup_hash=$(sha256sum "$backup" | awk '{print $1}')
 printf '%s  %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256"
 printf 'environment=production\n' >"$backup.metadata"
-chmod 600 "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata"
+chmod 600 "$bundle" "$bundle.sha256" "$image_archive" \
+  "$image_archive.sha256" "$image_manifest" "$backup" "$backup.sha256" \
+  "$backup.metadata"

 for script in validate-production-env.sh check-environment-readiness.sh \
  verify-realtime-cluster.sh verify-beam-runtime.sh; do
@ -86,10 +112,6 @@ shift
 case "$*" in
  'config --quiet') exit 0 ;;
  'ps -q app') printf 'app-1\n'; exit 0 ;;
-  'build migrate')
-    printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
-    exit 0
-    ;;
  'stop app')
    printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
    exit 0
@ -173,21 +195,30 @@ if [ "$1" = inspect ]; then
          "$MOCK_ENV_FILE"
      fi
      ;;
-    '{{.Image}}') printf 'sha256:mock-candidate\n' ;;
+    '{{.Image}}') printf '%s\n' "$MOCK_CANDIDATE_ID" ;;
    *) exit 1 ;;
  esac
  exit 0
 fi
 if [ "$1" = image ] && [ "$2" = inspect ] &&
-  [ "$3" = --format ] && [ "$4" = '{{.Id}}' ]; then
-  printf 'sha256:mock-candidate\n'
+  [ "$3" = --format ]; then
+  case "$4" in
+    '{{.Id}}') printf '%s\n' "$MOCK_CANDIDATE_ID" ;;
+    '{{.Os}}/{{.Architecture}}') printf 'linux/amd64\n' ;;
+    *) exit 1 ;;
+  esac
+  exit 0
+fi
+if [ "$1" = load ]; then
+  cat >/dev/null
+  printf 'docker:load\n' >>"$MOCK_COMMAND_LOG"
  exit 0
 fi
 printf 'Unexpected docker invocation: %s\n' "$*" >&2
 exit 1
 EOF

-for command in curl pg_restore; do
+for command in curl jq pg_restore; do
  install -m 755 /dev/null "$mock_bin/$command"
  printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command"
 done
@ -200,6 +231,7 @@ container_env=(
  --env "MOCK_GIT_STATE=$remote_root/git-state"
  --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log"
  --env "MOCK_ENV_FILE=$remote_root/.env"
+  --env "MOCK_CANDIDATE_ID=$mock_candidate_id"
  --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 )
 container_mounts=(
@ -230,7 +262,9 @@ run_release() {
      "$remote_root/output/releases/incoming/$(basename -- "$bundle")" \
      "$target_commit" \
      "$remote_root/output/backups/production/$(basename -- "$backup")" \
-      "$policy"
+      "$policy" \
+      "$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \
+      "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")"
 }

 run_release forward_only >"$run_dir/forward-success.out"
@ -238,7 +272,11 @@ grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
  "$fixture/.env" >/dev/null
 grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \
  "$fixture/.env" >/dev/null
-grep -F 'compose:build migrate' "$fixture/mock-commands.log" >/dev/null
+grep -Fx 'docker:load' "$fixture/mock-commands.log" >/dev/null
+if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then
+  echo "Production release drill unexpectedly compiled on the production host." >&2
+  exit 1
+fi
 grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null
 grep -F 'compose:run --rm --no-deps --interactive=false migrate' \
  "$fixture/mock-commands.log" >/dev/null
--- a/scripts/production-release-remote.sh
+++ b/scripts/production-release-remote.sh
@ -9,10 +9,12 @@ bundle=${4:-}
 target_commit=${5:-}
 backup=${6:-}
 expected_migration_policy=${7:-}
+image_archive=${8:-}
+image_manifest=${9:-}

 usage() {
  echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2
-  echo "       $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY" >&2
+  echo "       $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2
 }

 case "$action" in
@ -26,6 +28,13 @@ if [[ "$root" != "/srv/who_need_help-production" ]]; then
  exit 2
 fi

+for command in curl docker git gzip jq pg_restore sha256sum; do
+  command -v "$command" >/dev/null 2>&1 || {
+    echo "Required production command is unavailable: $command" >&2
+    exit 2
+  }
+done
+
 env_file="$root/.env"
 if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then
  echo "Production .env is missing or does not have mode 0600." >&2
@ -122,7 +131,8 @@ if [[ "$action" == "plan" ]]; then
 fi

 if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
-  -z "$expected_migration_policy" ]]; then
+  -z "$expected_migration_policy" || -z "$image_archive" ||
+  -z "$image_manifest" ]]; then
  usage
  exit 2
 fi
@ -133,7 +143,9 @@ if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$expected_confirmation" ]]; then
  exit 2
 fi

-for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata"; do
+for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
+  "$backup.metadata" "$image_archive" "$image_archive.sha256" \
+  "$image_manifest"; do
  [[ -f "$required_file" ]] || {
    echo "Required release evidence is missing: $required_file" >&2
    exit 2
@ -149,6 +161,14 @@ done
  sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
 )
 pg_restore --list "$backup" >/dev/null
+(
+  cd "$(dirname -- "$image_archive")"
+  sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null
+)
+gzip -t "$image_archive"
+grep -Fx 'format=1' "$image_manifest" >/dev/null
+grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
+grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
 git -C "$root" bundle verify "$bundle" >/dev/null

 bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
@ -277,14 +297,8 @@ rollback_runtime() {
  exit "$status"
 }
 case "$app_topology" in
-  compact)
-    build_services=(migrate)
-    runtime_services=(app)
-    ;;
-  split)
-    build_services=(docker-api-proxy proxy migrate)
-    runtime_services=(docker-api-proxy proxy web worker)
-    ;;
+  compact) runtime_services=(app) ;;
+  split) runtime_services=(docker-api-proxy proxy web worker) ;;
 esac

 verify_candidate_runtime() {
@ -344,7 +358,42 @@ revision_changed=true
 "$root/scripts/check-environment-readiness.sh" \
  "$env_file" --require-server-release

-"$root/scripts/compose.sh" "$env_file" build "${build_services[@]}"
+expected_images=("$(read_value APP_IMAGE)")
+if [[ "$app_topology" == split ]]; then
+  expected_images+=(
+    "$(read_value SOCKET_PROXY_IMAGE)"
+    "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')"
+  )
+fi
+
+grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null
+test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}"
+for image in "${expected_images[@]}"; do
+  awk -F'|' -v image="$image" '
+    $1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 }
+    END { if (!found) exit 1 }
+  ' "$image_manifest"
+done
+
+gzip -dc "$image_archive" | docker load >/dev/null
+for image in "${expected_images[@]}"; do
+  expected_id=$(
+    awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' \
+      "$image_manifest"
+  )
+  [[ -n "$expected_id" ]] || {
+    echo "Image manifest is missing the expected image ID: $image" >&2
+    exit 2
+  }
+  [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || {
+    echo "Loaded image ID does not match the signed manifest: $image" >&2
+    exit 2
+  }
+  [[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || {
+    echo "Loaded production image is not linux/amd64: $image" >&2
+    exit 2
+  }
+done

 if [[ "$migration_policy" == "forward_only" ]]; then
  echo "Stopping the old application before the forward-only migration boundary."
--- a/scripts/production-release.sh
+++ b/scripts/production-release.sh
@ -15,7 +15,7 @@ case "$action" in
    ;;
 esac

-for command in git pg_restore scp sha256sum ssh; do
+for command in docker git gzip mktemp pg_restore scp sha256sum ssh; do
  command -v "$command" >/dev/null 2>&1 || {
    echo "Required command is unavailable: $command" >&2
    exit 2
@ -127,14 +127,34 @@ fi
 "$ROOT/scripts/prepare-production-release.sh"
 release_dir="$ROOT/output/releases/$local_commit"
 bundle="$release_dir/who_need_help-$local_commit.bundle"
+image_archive="$release_dir/who_need_help-$local_commit-images-linux-amd64.tar.gz"
+image_checksum="$image_archive.sha256"
+image_manifest="$release_dir/who_need_help-$local_commit-images.manifest"
+
+production_env=$(mktemp)
+cleanup_production_env() {
+  trap - EXIT HUP INT TERM
+  rm -f "$production_env"
+}
+trap cleanup_production_env EXIT HUP INT TERM
+scp -p "$ssh_target:$remote_root/.env" "$production_env"
+chmod 600 "$production_env"
+"$ROOT/scripts/prepare-production-images.sh" "$production_env"
+cleanup_production_env
+
 remote_release_dir="$remote_root/output/releases/incoming"
 remote_bundle="$remote_release_dir/$(basename -- "$bundle")"
+remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")"
+remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")"
 timestamp=$(date -u +%Y%m%dT%H%M%SZ)
 remote_backup="$remote_root/output/backups/production/pre-$timestamp-${local_commit:0:12}.dump"

 ssh -o BatchMode=yes "$ssh_target" \
  "install -d -m 700 '$remote_release_dir'"
-scp -p "$bundle" "$bundle.sha256" "$ssh_target:$remote_release_dir/"
+scp -p \
+  "$bundle" "$bundle.sha256" \
+  "$image_archive" "$image_checksum" "$image_manifest" \
+  "$ssh_target:$remote_release_dir/"

 ssh -o BatchMode=yes "$ssh_target" \
  "bash -s -- '$remote_root/.env' '$remote_backup' production" \
@ -160,7 +180,7 @@ quoted_forward_confirmation=$(
  printf '%q' "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}"
 )
 ssh -o BatchMode=yes "$ssh_target" \
-  "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy'" \
+  "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \
  <"$ROOT/scripts/production-release-remote.sh"

 echo "Production release and public health verification completed."