Add verified production application rollback

2026-07-23 22:48:48 +03:00 · 2026-07-23 22:48:48 +03:00 · 564897cd3a
commit 564897cd3a
parent 0d880fc21e
7 changed files with 744 additions and 4 deletions
--- a/docs/operations.md
+++ b/docs/operations.md
@ -883,5 +883,41 @@ rollback and database migration rollback are separate decisions: do not run an
 Ecto down migration merely because an image is rolled back. Inspect the exact
 migration and compatibility boundary first.

-The repository intentionally does not ship an automatic destructive production
+Every successful SSH release prints a mode-`0600` `rollback-manifest.txt`.
+Before changing production, inspect that exact manifest with the read-only
+rollback plan:
+
+```bash
+./scripts/production-rollback.sh plan \
+  /srv/who_need_help-production/output/releases/RELEASE_ID/rollback-manifest.txt \
+  whoneedhelp
+```
+
+The plan requires the manifest target to be the currently checked-out
+production commit, verifies the previous immutable application and edge images
+still exist, checks the pre-release backup catalog and checksum, and prints the
+exact confirmation token. It does not change the remote environment or
+containers.
+
+After separately reviewing application/schema backward compatibility and
+approving that exact scope, run:
+
+```bash
+WNH_PRODUCTION_ROLLBACK_CONFIRM=whoneedhelp.com:TARGET_COMMIT:PREVIOUS_COMMIT \
+  ./scripts/production-rollback.sh apply \
+  /srv/who_need_help-production/output/releases/RELEASE_ID/rollback-manifest.txt \
+  whoneedhelp
+```
+
+This application rollback atomically restores the four previous image
+selectors and recreates only the selected application topology and shared edge
+with `--no-build`. It verifies the resulting image identities, container
+health, public readiness, and App Links. A failed rollback attempts to restore
+the pre-rollback image selection. The Git checkout intentionally remains at
+the newer source commit so the reviewed release tooling and manifest remain
+available.
+
+The command never restores PostgreSQL, reverses Ecto migrations, changes the
+test deployment, or touches the public Git/Devpost submission. The repository
+intentionally does not ship an automatic destructive production database
 restore command.
--- a/docs/verification.md
+++ b/docs/verification.md
@ -13,7 +13,7 @@ edit, branch, or tag was made during this audit.
  configured threshold, actionlint, every Compose render, Prometheus and
  Alertmanager validation, Helm lint, Trivy source and image scans, formatting,
  compilation, xref, strict Credo, Sobelow, Dialyzer, Hex audit, both npm audits,
-  and all 341 ExUnit tests. The configured image scans reported zero
+  and all 352 ExUnit tests. The configured image scans reported zero
  vulnerabilities, and the run left no project-scoped quality containers,
  networks, volumes, or one-run image tags.
 - A dedicated Brevo SMTP key and verified sender
@ -33,6 +33,21 @@ edit, branch, or tag was made during this audit.
  the Google OAuth client pair, the four public Firebase Android values, and the
  FCM service-account credential. No release-readiness claim is made until those
  credentials are imported and provider/device behavior is exercised.
+- The SSH production release `plan` action was repeated read-only. It observed
+  production commit `921e04b3608007675e22e7e26e0beb3975dbba58`, compact
+  topology, external PostgreSQL 18.4, healthy application containers, and
+  passing public readiness. The plan correctly refused release because the
+  production checkout still lacks five Android/push capability groups. It
+  reported 38 pending local commits and made no remote change.
+- A separate manual application rollback command now consumes only a successful
+  release's mode-`0600` manifest. Its plan verifies current/previous commits,
+  old application/edge images, backup checksum/catalog, runtime identity, and
+  public health. Apply requires an exact target/previous-commit confirmation,
+  atomically restores four image selectors, and recreates only the active
+  application topology and edge with `--no-deps --no-build`; Git, migrations,
+  the database, test, public Git, and Devpost are excluded. An isolated offline
+  fixture passed read-only plan, successful apply, and injected-edge-failure
+  recovery, including restoration of the original image selection.
 - Two web and two worker replicas, PostGIS, Mailpit, Traefik, and the scoped
  Docker socket proxy were running after the audit. Both web replicas and both
  workers were healthy; public liveness and readiness returned `ok` and
--- a/scripts/production-release-remote.sh
+++ b/scripts/production-release-remote.sh
@ -212,7 +212,7 @@ rollback_runtime() {
    echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2
    restore_image_revision
    "$root/scripts/compose.sh" "$env_file" \
-      up -d --no-build --wait --remove-orphans || true
+      up -d --no-deps --no-build --wait "${expected_services[@]}" || true

    edge_project=$(read_value EDGE_COMPOSE_PROJECT_NAME)
    docker compose \
@ -220,7 +220,7 @@ rollback_runtime() {
      --project-directory "$root" \
      --env-file "$env_file" \
      --file "$root/compose.edge.yaml" \
-      up -d --no-build --wait --remove-orphans || true
+      up -d --no-deps --no-build --wait edge || true

    curl --fail --silent --show-error --max-time 15 \
      "https://$expected_domain/healthz/ready" >/dev/null || true
--- a/scripts/production-rollback-drill.sh
+++ b/scripts/production-rollback-drill.sh
@ -0,0 +1,252 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd"
+run_dir=$(mktemp -d "$ROOT/output/rollback-drill.XXXXXX")
+fixture="$run_dir/production"
+mock_bin="$run_dir/mock-bin"
+remote_root=/srv/who_need_help-production
+manifest="$remote_root/output/releases/release-1/rollback-manifest.txt"
+target_commit=2222222222222222222222222222222222222222
+previous_commit=1111111111111111111111111111111111111111
+confirmation="whoneedhelp.com:$target_commit:$previous_commit"
+
+cleanup() {
+  trap - EXIT HUP INT TERM
+  find "$run_dir" -xdev -depth -delete 2>/dev/null || true
+}
+trap cleanup EXIT HUP INT TERM
+
+install -d -m 700 \
+  "$fixture/.git" \
+  "$fixture/scripts" \
+  "$fixture/output/releases/release-1" \
+  "$fixture/output/backups/production" \
+  "$mock_bin"
+
+install -m 600 /dev/null "$fixture/.env"
+printf '%s\n' \
+  'DEPLOYMENT_ENV=production' \
+  'COMPOSE_PROJECT_NAME=who_need_help_production' \
+  'DATABASE_MODE=external' \
+  'APP_TOPOLOGY=compact' \
+  'PHX_HOST=whoneedhelp.com' \
+  'WNH_BASE_URL=https://whoneedhelp.com' \
+  'EDGE_COMPOSE_PROJECT_NAME=who_need_help_edge' \
+  "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
+  "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}" \
+  "POSTGIS_IMAGE=who-need-help:postgis-production-${target_commit:0:12}" \
+  "CADDY_IMAGE=who-need-help:caddy-production-${target_commit:0:12}" \
+  >"$fixture/.env"
+
+backup="$fixture/output/backups/production/pre-release.dump"
+printf 'isolated rollback drill backup\n' >"$backup"
+backup_hash=$(sha256sum "$backup" | awk '{print $1}')
+printf '%s  %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256"
+printf 'environment=production\n' >"$backup.metadata"
+chmod 600 "$backup" "$backup.sha256" "$backup.metadata"
+
+install -m 600 /dev/null "$fixture/output/releases/release-1/rollback-manifest.txt"
+printf '%s\n' \
+  "previous_commit=$previous_commit" \
+  "target_commit=$target_commit" \
+  "APP_IMAGE=who-need-help:production-${previous_commit:0:12}" \
+  "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${previous_commit:0:12}" \
+  "POSTGIS_IMAGE=who-need-help:postgis-production-${previous_commit:0:12}" \
+  "CADDY_IMAGE=who-need-help:caddy-production-${previous_commit:0:12}" \
+  "database_backup=$remote_root/output/backups/production/pre-release.dump" \
+  'status=started' \
+  'status=success' \
+  >"$fixture/output/releases/release-1/rollback-manifest.txt"
+
+install -m 755 /dev/null "$fixture/scripts/validate-production-env.sh"
+printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/validate-production-env.sh"
+
+install -m 755 /dev/null "$fixture/scripts/compose.sh"
+cat >"$fixture/scripts/compose.sh" <<'EOF'
+#!/bin/sh
+set -eu
+env_file=$1
+shift
+case "$*" in
+  'config --quiet') exit 0 ;;
+  'ps -q app') printf 'app-1\n'; exit 0 ;;
+  up\ *) printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"; exit 0 ;;
+esac
+printf 'Unexpected compose invocation: %s\n' "$*" >&2
+exit 1
+EOF
+
+install -m 755 /dev/null "$mock_bin/git"
+cat >"$mock_bin/git" <<'EOF'
+#!/bin/sh
+set -eu
+case " $* " in
+  *' status --porcelain --untracked-files=no '*) exit 0 ;;
+  *' rev-parse --verify HEAD '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
+  *' cat-file -e '*) exit 0 ;;
+  *' merge-base --is-ancestor '*) exit 0 ;;
+esac
+printf 'Unexpected git invocation: %s\n' "$*" >&2
+exit 1
+EOF
+
+install -m 755 /dev/null "$mock_bin/docker"
+cat >"$mock_bin/docker" <<'EOF'
+#!/bin/sh
+set -eu
+if [ "$1" = image ] && [ "$2" = inspect ]; then
+  exit 0
+fi
+if [ "$1" = inspect ]; then
+  format=$3
+  container=$4
+  case "$format" in
+    '{{.State.Status}}') printf 'running\n' ;;
+    '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;;
+    '{{.Config.Image}}')
+      case "$container" in
+        app-1) awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' "$MOCK_ENV_FILE" ;;
+        edge-1) awk -F= '$1 == "CADDY_IMAGE" {print substr($0, index($0, "=") + 1)}' "$MOCK_ENV_FILE" ;;
+        *) exit 1 ;;
+      esac
+      ;;
+    *) exit 1 ;;
+  esac
+  exit 0
+fi
+if [ "$1" = compose ]; then
+  case " $* " in
+    *' ps -q edge ') printf 'edge-1\n'; exit 0 ;;
+    *' up -d --no-deps --no-build --wait edge ')
+      if [ "${MOCK_FAIL_EDGE_UP:-}" = once ] &&
+        [ ! -e "$MOCK_FAIL_EDGE_MARKER" ]; then
+        : >"$MOCK_FAIL_EDGE_MARKER"
+        exit 17
+      fi
+      printf 'edge:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
+      exit 0
+      ;;
+  esac
+fi
+printf 'Unexpected docker invocation: %s\n' "$*" >&2
+exit 1
+EOF
+
+for command in curl pg_restore; do
+  install -m 755 /dev/null "$mock_bin/$command"
+  printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command"
+done
+
+touch "$fixture/mock-commands.log"
+chmod 600 "$fixture/mock-commands.log"
+
+container_env=(
+  --env "MOCK_TARGET_COMMIT=$target_commit"
+  --env "MOCK_ENV_FILE=$remote_root/.env"
+  --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log"
+  --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+)
+container_mounts=(
+  --volume "$fixture:$remote_root"
+  --volume "$mock_bin:/mock-bin:ro"
+  --volume "$ROOT/scripts/production-rollback-remote.sh:/runner/production-rollback-remote.sh:ro"
+)
+
+docker run --rm \
+  --network none \
+  --user "$(id -u):$(id -g)" \
+  --read-only \
+  --tmpfs /tmp:rw,nosuid,nodev,noexec \
+  --cap-drop ALL \
+  --security-opt no-new-privileges \
+  "${container_env[@]}" \
+  "${container_mounts[@]}" \
+  "$BASE_IMAGE" \
+  bash /runner/production-rollback-remote.sh \
+    plan "$remote_root" whoneedhelp.com "$manifest" \
+    >"$run_dir/plan.out"
+
+grep -F "Exact confirmation: $confirmation" "$run_dir/plan.out" >/dev/null
+grep -F 'Read-only production application rollback scope check passed.' \
+  "$run_dir/plan.out" >/dev/null
+test ! -s "$fixture/mock-commands.log"
+
+docker run --rm \
+  --network none \
+  --user "$(id -u):$(id -g)" \
+  --read-only \
+  --tmpfs /tmp:rw,nosuid,nodev,noexec \
+  --cap-drop ALL \
+  --security-opt no-new-privileges \
+  --env "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation" \
+  "${container_env[@]}" \
+  "${container_mounts[@]}" \
+  "$BASE_IMAGE" \
+  bash /runner/production-rollback-remote.sh \
+    apply "$remote_root" whoneedhelp.com "$manifest" \
+    >"$run_dir/apply.out"
+
+grep -Fx "APP_IMAGE=who-need-help:production-${previous_commit:0:12}" \
+  "$fixture/.env" >/dev/null
+grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${previous_commit:0:12}" \
+  "$fixture/.env" >/dev/null
+grep -F 'compose:up -d --no-deps --no-build --wait app' \
+  "$fixture/mock-commands.log" >/dev/null
+grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null
+if grep -E -- 'migrate|--build|(^|[[:space:]])db([[:space:]]|$)' \
+  "$fixture/mock-commands.log" >/dev/null; then
+  echo "Rollback drill touched migrations, builds, or the database service." >&2
+  exit 1
+fi
+find "$fixture/output/releases/release-1" \
+  -maxdepth 1 -type f -name 'application-rollback-*.txt' -print -quit |
+  grep -q .
+grep -F "Production application images rolled back to release $previous_commit." \
+  "$run_dir/apply.out" >/dev/null
+
+sed -i \
+  -e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:production-${target_commit:0:12}|" \
+  -e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}|" \
+  -e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-production-${target_commit:0:12}|" \
+  -e "s|^CADDY_IMAGE=.*|CADDY_IMAGE=who-need-help:caddy-production-${target_commit:0:12}|" \
+  "$fixture/.env"
+: >"$fixture/mock-commands.log"
+
+set +e
+docker run --rm \
+  --network none \
+  --user "$(id -u):$(id -g)" \
+  --read-only \
+  --tmpfs /tmp:rw,nosuid,nodev,noexec \
+  --cap-drop ALL \
+  --security-opt no-new-privileges \
+  --env "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation" \
+  --env MOCK_FAIL_EDGE_UP=once \
+  --env "MOCK_FAIL_EDGE_MARKER=$remote_root/mock-edge-failed-once" \
+  "${container_env[@]}" \
+  "${container_mounts[@]}" \
+  "$BASE_IMAGE" \
+  bash /runner/production-rollback-remote.sh \
+    apply "$remote_root" whoneedhelp.com "$manifest" \
+    >"$run_dir/failure.out" 2>&1
+failure_status=$?
+set -e
+
+if [[ "$failure_status" -eq 0 ]]; then
+  echo "Rollback drill did not surface the injected edge failure." >&2
+  exit 1
+fi
+grep -F 'Rollback failed; restoring the pre-rollback image selection.' \
+  "$run_dir/failure.out" >/dev/null
+grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \
+  "$fixture/.env" >/dev/null
+grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${target_commit:0:12}" \
+  "$fixture/.env" >/dev/null
+test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \
+  "$fixture/mock-commands.log")" = 2
+grep -F 'edge:' "$fixture/mock-commands.log" >/dev/null
+
+echo "Isolated production application rollback plan/apply/failure-recovery drill passed."
--- a/scripts/production-rollback-remote.sh
+++ b/scripts/production-rollback-remote.sh
@ -0,0 +1,364 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+action=${1:-}
+root=${2:-/srv/who_need_help-production}
+expected_domain=${3:-whoneedhelp.com}
+manifest=${4:-}
+
+usage() {
+  echo "Usage: $0 plan|apply /srv/who_need_help-production whoneedhelp.com ROLLBACK_MANIFEST" >&2
+}
+
+case "$action" in
+  plan | apply) ;;
+  *) usage; exit 2 ;;
+esac
+
+root=$(realpath --canonicalize-existing "$root")
+if [[ "$root" != "/srv/who_need_help-production" ]]; then
+  echo "Refusing a production rollback outside /srv/who_need_help-production." >&2
+  exit 2
+fi
+
+if [[ -z "$manifest" ]]; then
+  usage
+  exit 2
+fi
+
+manifest=$(realpath --canonicalize-existing "$manifest")
+case "$manifest" in
+  "$root"/output/releases/*/rollback-manifest.txt) ;;
+  *)
+    echo "Rollback manifest must be below $root/output/releases/." >&2
+    exit 2
+    ;;
+esac
+
+env_file="$root/.env"
+if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then
+  echo "Production .env is missing or does not have mode 0600." >&2
+  exit 2
+fi
+if [[ "$(stat -c '%a' "$manifest")" != 600 ]]; then
+  echo "Rollback manifest must have mode 0600." >&2
+  exit 2
+fi
+
+read_unique() {
+  local file=$1 key=$2 count
+  count=$(awk -F= -v key="$key" '$1 == key {count++} END {print count + 0}' "$file")
+  if [[ "$count" -ne 1 ]]; then
+    echo "$key must occur exactly once in $file." >&2
+    exit 2
+  fi
+  awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1)}' "$file"
+}
+
+read_last() {
+  local file=$1 key=$2
+  awk -F= -v key="$key" '
+    $1 == key {value = substr($0, index($0, "=") + 1); found = 1}
+    END {if (!found) exit 1; print value}
+  ' "$file"
+}
+
+require_commit() {
+  local value=$1 label=$2
+  [[ "$value" =~ ^[0-9a-f]{40}$ ]] || {
+    echo "$label is not a full Git commit." >&2
+    exit 2
+  }
+}
+
+require_image() {
+  local value=$1 prefix=$2 label=$3
+  [[ "$value" =~ ^who-need-help:${prefix}[A-Za-z0-9_.-]+$ ]] || {
+    echo "$label is not an expected immutable Who Need Help image tag." >&2
+    exit 2
+  }
+}
+
+deployment_environment=$(read_unique "$env_file" DEPLOYMENT_ENV)
+compose_project=$(read_unique "$env_file" COMPOSE_PROJECT_NAME)
+database_mode=$(read_unique "$env_file" DATABASE_MODE)
+app_topology=$(read_unique "$env_file" APP_TOPOLOGY)
+phx_host=$(read_unique "$env_file" PHX_HOST)
+public_origin=$(read_unique "$env_file" WNH_BASE_URL)
+edge_project=$(read_unique "$env_file" EDGE_COMPOSE_PROJECT_NAME)
+
+[[ "$deployment_environment" == production ]] || {
+  echo "DEPLOYMENT_ENV is not production." >&2
+  exit 2
+}
+[[ "$compose_project" == who_need_help_production ]] || {
+  echo "Unexpected production Compose project." >&2
+  exit 2
+}
+[[ "$database_mode" == external ]] || {
+  echo "The verified production rollback workflow expects DATABASE_MODE=external." >&2
+  exit 2
+}
+[[ "$phx_host" == "$expected_domain" &&
+  "$public_origin" == "https://$expected_domain" ]] || {
+  echo "Production origin does not match the expected domain." >&2
+  exit 2
+}
+[[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || {
+  echo "Production checkout has tracked modifications." >&2
+  exit 2
+}
+
+"$root/scripts/validate-production-env.sh" "$env_file" "$expected_domain" >/dev/null
+"$root/scripts/compose.sh" "$env_file" config --quiet
+
+previous_commit=$(read_unique "$manifest" previous_commit)
+target_commit=$(read_unique "$manifest" target_commit)
+backup=$(read_unique "$manifest" database_backup)
+release_status=$(read_last "$manifest" status)
+require_commit "$previous_commit" previous_commit
+require_commit "$target_commit" target_commit
+
+[[ "$release_status" == success ]] || {
+  echo "Only a manifest from a successful release can drive a manual rollback." >&2
+  exit 2
+}
+
+current_commit=$(git -C "$root" rev-parse --verify HEAD)
+[[ "$current_commit" == "$target_commit" ]] || {
+  echo "The manifest target is not the currently checked-out production commit." >&2
+  exit 2
+}
+git -C "$root" cat-file -e "$previous_commit^{commit}"
+git -C "$root" merge-base --is-ancestor "$previous_commit" "$target_commit" || {
+  echo "The manifest does not describe a forward production release." >&2
+  exit 2
+}
+
+previous_app_image=$(read_unique "$manifest" APP_IMAGE)
+previous_socket_image=$(read_unique "$manifest" SOCKET_PROXY_IMAGE)
+previous_postgis_image=$(read_unique "$manifest" POSTGIS_IMAGE)
+previous_caddy_image=$(read_unique "$manifest" CADDY_IMAGE)
+require_image "$previous_app_image" production- APP_IMAGE
+require_image "$previous_socket_image" socket-proxy-production- SOCKET_PROXY_IMAGE
+require_image "$previous_postgis_image" postgis-production- POSTGIS_IMAGE
+require_image "$previous_caddy_image" caddy-production- CADDY_IMAGE
+
+target_short=${target_commit:0:12}
+current_app_image=$(read_unique "$env_file" APP_IMAGE)
+current_socket_image=$(read_unique "$env_file" SOCKET_PROXY_IMAGE)
+current_postgis_image=$(read_unique "$env_file" POSTGIS_IMAGE)
+current_caddy_image=$(read_unique "$env_file" CADDY_IMAGE)
+
+[[ "$current_app_image" == "who-need-help:production-$target_short" &&
+  "$current_socket_image" == "who-need-help:socket-proxy-production-$target_short" &&
+  "$current_postgis_image" == "who-need-help:postgis-production-$target_short" &&
+  "$current_caddy_image" == "who-need-help:caddy-production-$target_short" ]] || {
+  echo "Current production image selection does not match the manifest target commit." >&2
+  exit 2
+}
+
+for image in "$previous_app_image" "$previous_caddy_image"; do
+  docker image inspect "$image" >/dev/null
+done
+
+backup=$(realpath --canonicalize-existing "$backup")
+case "$backup" in
+  "$root"/output/backups/production/*.dump) ;;
+  *)
+    echo "Manifest backup is outside the production backup directory." >&2
+    exit 2
+    ;;
+esac
+for required_file in "$backup" "$backup.sha256" "$backup.metadata"; do
+  [[ -f "$required_file" ]] || {
+    echo "Required rollback evidence is missing: $required_file" >&2
+    exit 2
+  }
+done
+(
+  cd "$(dirname -- "$backup")"
+  sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
+)
+pg_restore --list "$backup" >/dev/null
+
+case "$app_topology" in
+  compact) app_services=(app) ;;
+  split) app_services=(web worker) ;;
+  *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;;
+esac
+
+check_application() {
+  local expected_image=$1 service container state health image
+  for service in "${app_services[@]}"; do
+    mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
+    [[ ${#containers[@]} -gt 0 ]] || {
+      echo "Production service is not running: $service" >&2
+      return 1
+    }
+    for container in "${containers[@]}"; do
+      state=$(docker inspect --format '{{.State.Status}}' "$container")
+      health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
+      image=$(docker inspect --format '{{.Config.Image}}' "$container")
+      [[ "$state" == running && "$health" == healthy && "$image" == "$expected_image" ]] || {
+        echo "Production service does not match the expected healthy image: $service" >&2
+        return 1
+      }
+    done
+  done
+}
+
+edge_compose=(
+  docker compose
+  --project-name "$edge_project"
+  --project-directory "$root"
+  --env-file "$env_file"
+  --file "$root/compose.edge.yaml"
+)
+
+check_edge() {
+  local expected_image=$1 container state health image
+  mapfile -t containers < <("${edge_compose[@]}" ps -q edge)
+  [[ ${#containers[@]} -gt 0 ]] || {
+    echo "Production edge service is not running." >&2
+    return 1
+  }
+  for container in "${containers[@]}"; do
+    state=$(docker inspect --format '{{.State.Status}}' "$container")
+    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
+    image=$(docker inspect --format '{{.Config.Image}}' "$container")
+    [[ "$state" == running && "$health" == healthy && "$image" == "$expected_image" ]] || {
+      echo "Production edge does not match the expected healthy image." >&2
+      return 1
+    }
+  done
+}
+
+check_application "$current_app_image"
+check_edge "$current_caddy_image"
+curl --fail --silent --show-error --max-time 15 \
+  "https://$expected_domain/healthz/ready" >/dev/null
+
+confirmation="$expected_domain:$target_commit:$previous_commit"
+printf 'Production checkout: %s\n' "$root"
+printf 'Current source commit (unchanged by rollback): %s\n' "$target_commit"
+printf 'Application image rollback commit: %s\n' "$previous_commit"
+printf 'Compose project: %s\n' "$compose_project"
+printf 'Topology: %s\n' "$app_topology"
+printf 'Database mode: %s (no restore or migration reversal)\n' "$database_mode"
+printf 'Rollback manifest: %s\n' "$manifest"
+printf 'Verified backup evidence: %s\n' "$backup"
+printf 'Exact confirmation: %s\n' "$confirmation"
+echo "Scope: update four image selectors in production .env; recreate only application and edge containers."
+echo "Excluded: Git checkout, database, migrations, test deployment, public Git, and Devpost."
+
+if [[ "$action" == plan ]]; then
+  echo "Read-only production application rollback scope check passed."
+  exit 0
+fi
+
+if [[ "${WNH_PRODUCTION_ROLLBACK_CONFIRM:-}" != "$confirmation" ]]; then
+  echo "Set WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation for the approved rollback." >&2
+  exit 2
+fi
+
+update_images() {
+  local app_image=$1 socket_image=$2 postgis_image=$3 caddy_image=$4 temporary
+  temporary=$(mktemp "$root/.env.image-selection.XXXXXX")
+  chmod 600 "$temporary"
+
+  APP_IMAGE_VALUE=$app_image \
+  SOCKET_PROXY_IMAGE_VALUE=$socket_image \
+  POSTGIS_IMAGE_VALUE=$postgis_image \
+  CADDY_IMAGE_VALUE=$caddy_image \
+    awk '
+      BEGIN {
+        replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
+        replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
+        replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
+        replacement["CADDY_IMAGE"] = ENVIRON["CADDY_IMAGE_VALUE"]
+      }
+      {
+        separator = index($0, "=")
+        key = separator > 1 ? substr($0, 1, separator - 1) : ""
+        if (key in replacement) {
+          seen[key]++
+          print key "=" replacement[key]
+        } else {
+          print
+        }
+      }
+      END {
+        for (key in replacement) {
+          if (seen[key] != 1) exit 1
+        }
+      }
+    ' "$env_file" >"$temporary" || {
+      rm -f "$temporary"
+      return 1
+    }
+
+  mv "$temporary" "$env_file"
+  chmod 600 "$env_file"
+}
+
+runtime_changed=false
+recover_current_runtime() {
+  local status=$?
+  trap - EXIT HUP INT TERM
+  if [[ "$status" -ne 0 && "$runtime_changed" == true ]]; then
+    echo "Rollback failed; restoring the pre-rollback image selection." >&2
+    update_images \
+      "$current_app_image" \
+      "$current_socket_image" \
+      "$current_postgis_image" \
+      "$current_caddy_image" || true
+    "$root/scripts/compose.sh" "$env_file" \
+      up -d --no-deps --no-build --wait "${app_services[@]}" || true
+    "${edge_compose[@]}" \
+      up -d --no-deps --no-build --wait edge || true
+    curl --fail --silent --show-error --max-time 15 \
+      "https://$expected_domain/healthz/ready" >/dev/null || true
+  fi
+  exit "$status"
+}
+trap recover_current_runtime EXIT HUP INT TERM
+
+update_images \
+  "$previous_app_image" \
+  "$previous_socket_image" \
+  "$previous_postgis_image" \
+  "$previous_caddy_image"
+runtime_changed=true
+
+"$root/scripts/compose.sh" "$env_file" \
+  up -d --no-deps --no-build --wait "${app_services[@]}"
+"${edge_compose[@]}" \
+  up -d --no-deps --no-build --wait edge
+
+check_application "$previous_app_image"
+check_edge "$previous_caddy_image"
+curl --fail --silent --show-error --max-time 30 \
+  "https://$expected_domain/healthz/ready" >/dev/null
+curl --fail --silent --show-error --max-time 30 \
+  "https://$expected_domain/.well-known/assetlinks.json" >/dev/null
+
+audit_file="$(dirname -- "$manifest")/application-rollback-$(date -u +%Y%m%dT%H%M%SZ).txt"
+{
+  printf 'source_manifest=%s\n' "$manifest"
+  printf 'source_commit_retained=%s\n' "$target_commit"
+  printf 'application_images_restored_from_commit=%s\n' "$previous_commit"
+  printf 'database_action=none\n'
+  printf 'migration_action=none\n'
+  printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  printf 'status=success\n'
+} >"$audit_file"
+chmod 600 "$audit_file"
+
+runtime_changed=false
+trap - EXIT HUP INT TERM
+
+printf 'Production application images rolled back to release %s.\n' "$previous_commit"
+printf 'Production source remains at %s.\n' "$target_commit"
+printf 'Rollback audit: %s\n' "$audit_file"
--- a/scripts/production-rollback.sh
+++ b/scripts/production-rollback.sh
@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+action=${1:-plan}
+remote_manifest=${2:-}
+ssh_target=${3:-whoneedhelp}
+remote_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production}
+expected_domain=${WNH_PRODUCTION_DOMAIN:-whoneedhelp.com}
+
+case "$action" in
+  plan | apply) ;;
+  *)
+    echo "Usage: $0 [plan|apply] REMOTE_ROLLBACK_MANIFEST [SSH_TARGET]" >&2
+    exit 2
+    ;;
+esac
+
+if [[ -z "$remote_manifest" ]]; then
+  echo "Provide the absolute rollback-manifest.txt path printed by a successful release." >&2
+  exit 2
+fi
+
+case "$remote_manifest" in
+  "$remote_root"/output/releases/*/rollback-manifest.txt) ;;
+  *)
+    echo "Rollback manifest must be below $remote_root/output/releases/." >&2
+    exit 2
+    ;;
+esac
+
+command -v ssh >/dev/null 2>&1 || {
+  echo "Required command is unavailable: ssh" >&2
+  exit 2
+}
+
+quote() {
+  printf '%q' "$1"
+}
+
+remote_command() {
+  local remote_action=$1
+  printf 'bash -s -- %s %s %s %s' \
+    "$(quote "$remote_action")" \
+    "$(quote "$remote_root")" \
+    "$(quote "$expected_domain")" \
+    "$(quote "$remote_manifest")"
+}
+
+ssh -o BatchMode=yes "$ssh_target" \
+  "$(remote_command plan)" \
+  <"$ROOT/scripts/production-rollback-remote.sh"
+
+if [[ "$action" == plan ]]; then
+  echo "Production application rollback plan passed; no remote state was changed."
+  exit 0
+fi
+
+if [[ -z "${WNH_PRODUCTION_ROLLBACK_CONFIRM:-}" ]]; then
+  echo "Rollback execution requires the exact confirmation token printed by plan:" >&2
+  echo "WNH_PRODUCTION_ROLLBACK_CONFIRM=... $0 apply $remote_manifest $ssh_target" >&2
+  exit 2
+fi
+
+confirmation=$(quote "$WNH_PRODUCTION_ROLLBACK_CONFIRM")
+ssh -o BatchMode=yes "$ssh_target" \
+  "WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation $(remote_command apply)" \
+  <"$ROOT/scripts/production-rollback-remote.sh"
+
+echo "Production application rollback and public health verification completed."
--- a/scripts/quality.sh
+++ b/scripts/quality.sh
@ -64,6 +64,9 @@ docker run --rm \
  "$SHELLCHECK_IMAGE" \
  $(find scripts -type f -name '*.sh' -print | sort)

+echo "Checking isolated production application rollback plan/apply"
+./scripts/production-rollback-drill.sh
+
 echo "Checking Dockerfiles with Hadolint 2.14.0"
 for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \
  Dockerfile.postgis Dockerfile.socket-proxy Dockerfile.traefik \