#!/usr/bin/env bash set -euo pipefail umask 077 action=${1:-} root=${2:-/srv/who_need_help-production} expected_domain=${3:-whoneedhelp.com} bundle=${4:-} target_commit=${5:-} backup=${6:-} expected_migration_policy=${7:-} usage() { echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2 echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY" >&2 } case "$action" in plan | apply) ;; *) usage; exit 2 ;; esac root=$(realpath --canonicalize-existing "$root") if [[ "$root" != "/srv/who_need_help-production" ]]; then echo "Refusing a production release outside /srv/who_need_help-production." >&2 exit 2 fi env_file="$root/.env" if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then echo "Production .env is missing or does not have mode 0600." >&2 exit 2 fi read_value() { local key=$1 awk -v key="$key" ' index($0, key "=") == 1 { print substr($0, length(key) + 2) found = 1 exit } END { if (!found) exit 1 } ' "$env_file" } deployment_environment=$(read_value DEPLOYMENT_ENV) compose_project=$(read_value COMPOSE_PROJECT_NAME) database_mode=$(read_value DATABASE_MODE) app_topology=$(read_value APP_TOPOLOGY) phx_host=$(read_value PHX_HOST) public_origin=$(read_value WNH_BASE_URL) branch=$(git -C "$root" symbolic-ref --quiet --short HEAD || true) current_commit=$(git -C "$root" rev-parse --verify HEAD) [[ "$deployment_environment" == "production" ]] || { echo "DEPLOYMENT_ENV is not production." >&2 exit 2 } [[ "$compose_project" == "who_need_help_production" ]] || { echo "Unexpected production Compose project." >&2 exit 2 } [[ "$database_mode" == "external" ]] || { echo "The verified single-server production workflow expects DATABASE_MODE=external." >&2 exit 2 } [[ "$phx_host" == "$expected_domain" && "$public_origin" == "https://$expected_domain" ]] || { echo "Production origin does not match the expected domain." >&2 exit 2 } [[ "$branch" == "main" || -z "$branch" ]] || { echo "Production checkout must be on main or detached at the deployed commit." >&2 exit 2 } [[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || { echo "Production checkout has tracked modifications." >&2 exit 2 } "$root/scripts/compose.sh" "$env_file" config --quiet case "$app_topology" in compact) expected_services=(app) ;; split) expected_services=(web worker) ;; *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;; esac for service in "${expected_services[@]}"; do mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") [[ ${#containers[@]} -gt 0 ]] || { echo "Production service is not running: $service" >&2 exit 2 } for container in "${containers[@]}"; do state=$(docker inspect --format '{{.State.Status}}' "$container") health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") [[ "$state" == "running" && "$health" == "healthy" ]] || { echo "Production container is not healthy: $service" >&2 exit 2 } done done curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null printf 'Production checkout: %s\n' "$root" printf 'Current commit: %s\n' "$current_commit" printf 'Branch: %s\n' "${branch:-detached}" printf 'Compose project: %s\n' "$compose_project" printf 'Topology: %s\n' "$app_topology" printf 'Database mode: %s\n' "$database_mode" printf 'Public readiness: passed\n' df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root" if [[ "$action" == "plan" ]]; then echo "Read-only production release scope check passed." exit 0 fi if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" || -z "$expected_migration_policy" ]]; then usage exit 2 fi expected_confirmation="$expected_domain:$target_commit" if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$expected_confirmation" ]]; then echo "Set WNH_PRODUCTION_RELEASE_CONFIRM=$expected_confirmation for the approved release." >&2 exit 2 fi for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata"; do [[ -f "$required_file" ]] || { echo "Required release evidence is missing: $required_file" >&2 exit 2 } done ( cd "$(dirname -- "$bundle")" sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null ) ( cd "$(dirname -- "$backup")" sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null ) pg_restore --list "$backup" >/dev/null git -C "$root" bundle verify "$bundle" >/dev/null bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') [[ "$bundle_head" == "$target_commit" ]] || { echo "Bundle HEAD does not match the approved target commit." >&2 exit 2 } release_ref="refs/wnh/releases/$target_commit" git -C "$root" fetch "$bundle" "HEAD:$release_ref" [[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || { echo "Fetched release ref does not match the approved commit." >&2 exit 1 } git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { echo "Production updates must be a fast-forward from the deployed commit." >&2 exit 1 } policy_script=$(mktemp) trap 'rm -f "$policy_script"' EXIT HUP INT TERM git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script" chmod 600 "$policy_script" migration_policy_output=$( bash "$policy_script" "$current_commit" "$target_commit" "$root" ) rm -f "$policy_script" trap - EXIT HUP INT TERM printf '%s\n' "$migration_policy_output" migration_policy=$( awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output" ) [[ "$migration_policy" == "$expected_migration_policy" ]] || { echo "Remote migration policy does not match the locally approved policy." >&2 exit 2 } if [[ "$migration_policy" == "forward_only" ]]; then forward_confirmation="$expected_domain:$target_commit:forward-only" [[ "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || { echo "Set WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation for this schema boundary." >&2 exit 2 } fi release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}" release_dir="$root/output/releases/$release_id" mkdir -p "$release_dir" chmod 700 "$root/output" "$root/output/releases" "$release_dir" rollback_manifest="$release_dir/rollback-manifest.txt" { printf 'previous_commit=%s\n' "$current_commit" printf 'target_commit=%s\n' "$target_commit" printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)" printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)" printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)" printf 'migration_policy=%s\n' "$migration_policy" printf 'migration_details=%s\n' \ "$(awk -F= '$1 == "migration_versions" {print $2}' <<<"$migration_policy_output")" printf 'database_backup=%s\n' "$backup" printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'status=started\n' } >"$rollback_manifest" chmod 600 "$rollback_manifest" revision_changed=false migration_started=false restore_image_revision() { local temporary temporary=$(mktemp "$root/.env.release-rollback.XXXXXX") chmod 600 "$temporary" APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ awk ' BEGIN { replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"] replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"] replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"] } { separator = index($0, "=") key = separator > 1 ? substr($0, 1, separator - 1) : "" print (key in replacement) ? key "=" replacement[key] : $0 } ' "$env_file" >"$temporary" mv "$temporary" "$env_file" chmod 600 "$env_file" } rollback_runtime() { local status=$? trap - EXIT HUP INT TERM if [[ "$status" -ne 0 && "$revision_changed" == true && "$migration_policy" == "forward_only" && "$migration_started" == true ]]; then { printf 'status=forward-only-release-failed\n' printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'automatic_application_rollback=blocked\n' printf 'recovery_note=inspect migration state and repair the approved target release\n' } >>"$rollback_manifest" echo "Forward-only release failed after migration started." >&2 echo "The previous application will not be restarted against a potentially incompatible schema." >&2 elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2 restore_image_revision "$root/scripts/compose.sh" "$env_file" \ up -d --no-deps --no-build --wait "${runtime_services[@]}" || true curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null || true { printf 'status=runtime-rolled-back\n' printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'database_note=migrations_may_have_applied_and_were_not_reversed\n' } >>"$rollback_manifest" echo "The Git checkout and any applied migrations were intentionally not reversed automatically." >&2 fi exit "$status" } case "$app_topology" in compact) build_services=(migrate) runtime_services=(app) ;; split) build_services=(docker-api-proxy proxy migrate) runtime_services=(docker-api-proxy proxy web worker) ;; esac trap rollback_runtime EXIT HUP INT TERM if [[ "$branch" == "main" ]]; then git -C "$root" merge --ff-only "$release_ref" else git -C "$root" checkout --detach "$release_ref" fi "$root/scripts/set-deployment-revision.sh" "$env_file" --allow-pre-play revision_changed=true "$root/scripts/validate-production-env.sh" \ "$env_file" "$expected_domain" --allow-pre-play "$root/scripts/check-environment-readiness.sh" \ "$env_file" --require-server-release "$root/scripts/compose.sh" "$env_file" build "${build_services[@]}" if [[ "$migration_policy" == "forward_only" ]]; then echo "Stopping the old application before the forward-only migration boundary." "$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}" fi migration_started=true "$root/scripts/compose.sh" "$env_file" run --rm --no-deps migrate "$root/scripts/check-database.sh" "$env_file" "$root/scripts/compose.sh" "$env_file" \ up -d --no-deps --no-build --wait "${runtime_services[@]}" COMPOSE_PROJECT_NAME="$compose_project" \ "$root/scripts/verify-realtime-cluster.sh" compose COMPOSE_PROJECT_NAME="$compose_project" \ "$root/scripts/verify-beam-runtime.sh" compose curl --fail --silent --show-error --max-time 30 \ "https://$expected_domain/healthz/ready" >/dev/null curl --fail --silent --show-error --max-time 30 \ "https://$expected_domain/.well-known/assetlinks.json" >/dev/null { printf 'status=success\n' printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } >>"$rollback_manifest" revision_changed=false trap - EXIT HUP INT TERM printf 'Production release completed: %s\n' "$target_commit" printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest" printf 'Database backup: %s\n' "$backup"