#!/usr/bin/env bash set -euo pipefail umask 077 action=${1:-} root=${2:-/srv/who_need_help-production} expected_domain=${3:-whoneedhelp.com} manifest=${4:-} usage() { echo "Usage: $0 plan|apply /srv/who_need_help-production whoneedhelp.com ROLLBACK_MANIFEST" >&2 } case "$action" in plan | apply) ;; *) usage; exit 2 ;; esac root=$(realpath --canonicalize-existing "$root") if [[ "$root" != "/srv/who_need_help-production" ]]; then echo "Refusing a production rollback outside /srv/who_need_help-production." >&2 exit 2 fi if [[ -z "$manifest" ]]; then usage exit 2 fi manifest=$(realpath --canonicalize-existing "$manifest") case "$manifest" in "$root"/output/releases/*/rollback-manifest.txt) ;; *) echo "Rollback manifest must be below $root/output/releases/." >&2 exit 2 ;; esac env_file="$root/.env" if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then echo "Production .env is missing or does not have mode 0600." >&2 exit 2 fi if [[ "$(stat -c '%a' "$manifest")" != 600 ]]; then echo "Rollback manifest must have mode 0600." >&2 exit 2 fi read_unique() { local file=$1 key=$2 count count=$(awk -F= -v key="$key" '$1 == key {count++} END {print count + 0}' "$file") if [[ "$count" -ne 1 ]]; then echo "$key must occur exactly once in $file." >&2 exit 2 fi awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1)}' "$file" } read_last() { local file=$1 key=$2 awk -F= -v key="$key" ' $1 == key {value = substr($0, index($0, "=") + 1); found = 1} END {if (!found) exit 1; print value} ' "$file" } require_commit() { local value=$1 label=$2 [[ "$value" =~ ^[0-9a-f]{40}$ ]] || { echo "$label is not a full Git commit." >&2 exit 2 } } require_image() { local value=$1 prefix=$2 label=$3 [[ "$value" =~ ^who-need-help:${prefix}[A-Za-z0-9_.-]+$ ]] || { echo "$label is not an expected immutable Who Need Help image tag." >&2 exit 2 } } deployment_environment=$(read_unique "$env_file" DEPLOYMENT_ENV) compose_project=$(read_unique "$env_file" COMPOSE_PROJECT_NAME) database_mode=$(read_unique "$env_file" DATABASE_MODE) app_topology=$(read_unique "$env_file" APP_TOPOLOGY) phx_host=$(read_unique "$env_file" PHX_HOST) public_origin=$(read_unique "$env_file" WNH_BASE_URL) [[ "$deployment_environment" == production ]] || { echo "DEPLOYMENT_ENV is not production." >&2 exit 2 } [[ "$compose_project" == who_need_help_production ]] || { echo "Unexpected production Compose project." >&2 exit 2 } [[ "$database_mode" == external ]] || { echo "The verified production rollback workflow expects DATABASE_MODE=external." >&2 exit 2 } [[ "$phx_host" == "$expected_domain" && "$public_origin" == "https://$expected_domain" ]] || { echo "Production origin does not match the expected domain." >&2 exit 2 } [[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || { echo "Production checkout has tracked modifications." >&2 exit 2 } "$root/scripts/validate-production-env.sh" "$env_file" "$expected_domain" >/dev/null "$root/scripts/compose.sh" "$env_file" config --quiet previous_commit=$(read_unique "$manifest" previous_commit) target_commit=$(read_unique "$manifest" target_commit) backup=$(read_unique "$manifest" database_backup) release_status=$(read_last "$manifest" status) migration_policy=$(read_unique "$manifest" migration_policy) require_commit "$previous_commit" previous_commit require_commit "$target_commit" target_commit [[ "$release_status" == success ]] || { echo "Only a manifest from a successful release can drive a manual rollback." >&2 exit 2 } case "$migration_policy" in application_safe) ;; forward_only) echo "This release is marked forward_only; automatic old-image rollback is blocked." >&2 echo "Inspect the exact database migration state and use a forward repair." >&2 exit 2 ;; *) echo "The rollback manifest has an invalid migration policy." >&2 exit 2 ;; esac current_commit=$(git -C "$root" rev-parse --verify HEAD) [[ "$current_commit" == "$target_commit" ]] || { echo "The manifest target is not the currently checked-out production commit." >&2 exit 2 } git -C "$root" cat-file -e "$previous_commit^{commit}" git -C "$root" merge-base --is-ancestor "$previous_commit" "$target_commit" || { echo "The manifest does not describe a forward production release." >&2 exit 2 } previous_app_image=$(read_unique "$manifest" APP_IMAGE) previous_socket_image=$(read_unique "$manifest" SOCKET_PROXY_IMAGE) previous_postgis_image=$(read_unique "$manifest" POSTGIS_IMAGE) require_image "$previous_app_image" production- APP_IMAGE require_image "$previous_socket_image" socket-proxy-production- SOCKET_PROXY_IMAGE require_image "$previous_postgis_image" postgis-production- POSTGIS_IMAGE target_short=${target_commit:0:12} current_app_image=$(read_unique "$env_file" APP_IMAGE) current_socket_image=$(read_unique "$env_file" SOCKET_PROXY_IMAGE) current_postgis_image=$(read_unique "$env_file" POSTGIS_IMAGE) [[ "$current_app_image" == "who-need-help:production-$target_short" && "$current_socket_image" == "who-need-help:socket-proxy-production-$target_short" && "$current_postgis_image" == "who-need-help:postgis-production-$target_short" ]] || { echo "Current production image selection does not match the manifest target commit." >&2 exit 2 } docker image inspect "$previous_app_image" >/dev/null backup=$(realpath --canonicalize-existing "$backup") case "$backup" in "$root"/output/backups/production/*.dump) ;; *) echo "Manifest backup is outside the production backup directory." >&2 exit 2 ;; esac for required_file in "$backup" "$backup.sha256" "$backup.metadata"; do [[ -f "$required_file" ]] || { echo "Required rollback evidence is missing: $required_file" >&2 exit 2 } done ( cd "$(dirname -- "$backup")" sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null ) pg_restore --list "$backup" >/dev/null case "$app_topology" in compact) app_services=(app) ;; split) app_services=(web worker) ;; *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;; esac check_application() { local expected_image=$1 service container state health image for service in "${app_services[@]}"; do mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") [[ ${#containers[@]} -gt 0 ]] || { echo "Production service is not running: $service" >&2 return 1 } for container in "${containers[@]}"; do state=$(docker inspect --format '{{.State.Status}}' "$container") health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") image=$(docker inspect --format '{{.Config.Image}}' "$container") [[ "$state" == running && "$health" == healthy && "$image" == "$expected_image" ]] || { echo "Production service does not match the expected healthy image: $service" >&2 return 1 } done done } check_application "$current_app_image" curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null confirmation="$expected_domain:$target_commit:$previous_commit" printf 'Production checkout: %s\n' "$root" printf 'Current source commit (unchanged by rollback): %s\n' "$target_commit" printf 'Application image rollback commit: %s\n' "$previous_commit" printf 'Compose project: %s\n' "$compose_project" printf 'Topology: %s\n' "$app_topology" printf 'Database mode: %s (no restore or migration reversal)\n' "$database_mode" printf 'Rollback manifest: %s\n' "$manifest" printf 'Verified backup evidence: %s\n' "$backup" printf 'Exact confirmation: %s\n' "$confirmation" echo "Scope: update three application image selectors in production .env; recreate only application containers." echo "Excluded: shared edge/Caddy, Git checkout, database, migrations, test deployment, public Git, and Devpost." if [[ "$action" == plan ]]; then echo "Read-only production application rollback scope check passed." exit 0 fi if [[ "${WNH_PRODUCTION_ROLLBACK_CONFIRM:-}" != "$confirmation" ]]; then echo "Set WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation for the approved rollback." >&2 exit 2 fi update_images() { local app_image=$1 socket_image=$2 postgis_image=$3 temporary temporary=$(mktemp "$root/.env.image-selection.XXXXXX") chmod 600 "$temporary" APP_IMAGE_VALUE=$app_image \ SOCKET_PROXY_IMAGE_VALUE=$socket_image \ POSTGIS_IMAGE_VALUE=$postgis_image \ awk ' BEGIN { replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"] replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"] replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"] } { separator = index($0, "=") key = separator > 1 ? substr($0, 1, separator - 1) : "" if (key in replacement) { seen[key]++ print key "=" replacement[key] } else { print } } END { for (key in replacement) { if (seen[key] != 1) exit 1 } } ' "$env_file" >"$temporary" || { rm -f "$temporary" return 1 } mv "$temporary" "$env_file" chmod 600 "$env_file" } runtime_changed=false recover_current_runtime() { local status=$? trap - EXIT HUP INT TERM if [[ "$status" -ne 0 && "$runtime_changed" == true ]]; then echo "Rollback failed; restoring the pre-rollback image selection." >&2 update_images \ "$current_app_image" \ "$current_socket_image" \ "$current_postgis_image" || true "$root/scripts/compose.sh" "$env_file" \ up -d --no-deps --no-build --wait "${app_services[@]}" || true curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null || true fi exit "$status" } trap recover_current_runtime EXIT HUP INT TERM update_images \ "$previous_app_image" \ "$previous_socket_image" \ "$previous_postgis_image" runtime_changed=true "$root/scripts/compose.sh" "$env_file" \ up -d --no-deps --no-build --wait "${app_services[@]}" check_application "$previous_app_image" curl --fail --silent --show-error --max-time 30 \ "https://$expected_domain/healthz/ready" >/dev/null curl --fail --silent --show-error --max-time 30 \ "https://$expected_domain/.well-known/assetlinks.json" >/dev/null audit_file="$(dirname -- "$manifest")/application-rollback-$(date -u +%Y%m%dT%H%M%SZ).txt" { printf 'source_manifest=%s\n' "$manifest" printf 'source_commit_retained=%s\n' "$target_commit" printf 'application_images_restored_from_commit=%s\n' "$previous_commit" printf 'database_action=none\n' printf 'migration_action=none\n' printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'status=success\n' } >"$audit_file" chmod 600 "$audit_file" runtime_changed=false trap - EXIT HUP INT TERM printf 'Production application images rolled back to release %s.\n' "$previous_commit" printf 'Production source remains at %s.\n' "$target_commit" printf 'Rollback audit: %s\n' "$audit_file"