who_need_help/scripts/production-release-remote.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

action=${1:-}
root=${2:-/srv/who_need_help-production}
expected_domain=${3:-whoneedhelp.com}
bundle=${4:-}
target_commit=${5:-}
backup=${6:-}

usage() {
  echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2
  echo "       $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP" >&2
}

case "$action" in
  plan | apply) ;;
  *) usage; exit 2 ;;
esac

root=$(realpath --canonicalize-existing "$root")
if [[ "$root" != "/srv/who_need_help-production" ]]; then
  echo "Refusing a production release outside /srv/who_need_help-production." >&2
  exit 2
fi

env_file="$root/.env"
if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then
  echo "Production .env is missing or does not have mode 0600." >&2
  exit 2
fi

read_value() {
  local key=$1
  awk -v key="$key" '
    index($0, key "=") == 1 {
      print substr($0, length(key) + 2)
      found = 1
      exit
    }
    END { if (!found) exit 1 }
  ' "$env_file"
}

deployment_environment=$(read_value DEPLOYMENT_ENV)
compose_project=$(read_value COMPOSE_PROJECT_NAME)
database_mode=$(read_value DATABASE_MODE)
app_topology=$(read_value APP_TOPOLOGY)
phx_host=$(read_value PHX_HOST)
public_origin=$(read_value WNH_BASE_URL)
branch=$(git -C "$root" symbolic-ref --quiet --short HEAD || true)
current_commit=$(git -C "$root" rev-parse --verify HEAD)

[[ "$deployment_environment" == "production" ]] || {
  echo "DEPLOYMENT_ENV is not production." >&2
  exit 2
}
[[ "$compose_project" == "who_need_help_production" ]] || {
  echo "Unexpected production Compose project." >&2
  exit 2
}
[[ "$database_mode" == "external" ]] || {
  echo "The verified single-server production workflow expects DATABASE_MODE=external." >&2
  exit 2
}
[[ "$phx_host" == "$expected_domain" &&
  "$public_origin" == "https://$expected_domain" ]] || {
  echo "Production origin does not match the expected domain." >&2
  exit 2
}
[[ "$branch" == "main" || -z "$branch" ]] || {
  echo "Production checkout must be on main or detached at the deployed commit." >&2
  exit 2
}
[[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || {
  echo "Production checkout has tracked modifications." >&2
  exit 2
}

"$root/scripts/validate-production-env.sh" "$env_file" "$expected_domain" >/dev/null
"$root/scripts/compose.sh" "$env_file" config --quiet

case "$app_topology" in
  compact) expected_services=(app) ;;
  split) expected_services=(web worker) ;;
  *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;;
esac

for service in "${expected_services[@]}"; do
  mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
  [[ ${#containers[@]} -gt 0 ]] || {
    echo "Production service is not running: $service" >&2
    exit 2
  }

  for container in "${containers[@]}"; do
    state=$(docker inspect --format '{{.State.Status}}' "$container")
    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
    [[ "$state" == "running" && "$health" == "healthy" ]] || {
      echo "Production container is not healthy: $service" >&2
      exit 2
    }
  done
done

curl --fail --silent --show-error --max-time 15 \
  "https://$expected_domain/healthz/ready" >/dev/null

printf 'Production checkout: %s\n' "$root"
printf 'Current commit: %s\n' "$current_commit"
printf 'Branch: %s\n' "${branch:-detached}"
printf 'Compose project: %s\n' "$compose_project"
printf 'Topology: %s\n' "$app_topology"
printf 'Database mode: %s\n' "$database_mode"
printf 'Public readiness: passed\n'
df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root"

if [[ "$action" == "plan" ]]; then
  echo "Read-only production release scope check passed."
  exit 0
fi

if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ]]; then
  usage
  exit 2
fi

expected_confirmation="$expected_domain:$target_commit"
if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$expected_confirmation" ]]; then
  echo "Set WNH_PRODUCTION_RELEASE_CONFIRM=$expected_confirmation for the approved release." >&2
  exit 2
fi

for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata"; do
  [[ -f "$required_file" ]] || {
    echo "Required release evidence is missing: $required_file" >&2
    exit 2
  }
done

(
  cd "$(dirname -- "$bundle")"
  sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null
)
(
  cd "$(dirname -- "$backup")"
  sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
)
pg_restore --list "$backup" >/dev/null
git -C "$root" bundle verify "$bundle" >/dev/null

bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
[[ "$bundle_head" == "$target_commit" ]] || {
  echo "Bundle HEAD does not match the approved target commit." >&2
  exit 2
}

release_id="$(date -u +%Y%m%dT%H%M%SZ)-${target_commit:0:12}"
release_dir="$root/output/releases/$release_id"
mkdir -p "$release_dir"
chmod 700 "$root/output" "$root/output/releases" "$release_dir"
rollback_manifest="$release_dir/rollback-manifest.txt"

{
  printf 'previous_commit=%s\n' "$current_commit"
  printf 'target_commit=%s\n' "$target_commit"
  printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)"
  printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)"
  printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)"
  printf 'CADDY_IMAGE=%s\n' "$(read_value CADDY_IMAGE)"
  printf 'database_backup=%s\n' "$backup"
  printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  printf 'status=started\n'
} >"$rollback_manifest"
chmod 600 "$rollback_manifest"

revision_changed=false

restore_image_revision() {
  local temporary
  temporary=$(mktemp "$root/.env.release-rollback.XXXXXX")
  chmod 600 "$temporary"

  APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
  SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
  POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
  CADDY_IMAGE_VALUE=$(awk -F= '$1 == "CADDY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
    awk '
      BEGIN {
        replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
        replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
        replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
        replacement["CADDY_IMAGE"] = ENVIRON["CADDY_IMAGE_VALUE"]
      }
      {
        separator = index($0, "=")
        key = separator > 1 ? substr($0, 1, separator - 1) : ""
        print (key in replacement) ? key "=" replacement[key] : $0
      }
    ' "$env_file" >"$temporary"

  mv "$temporary" "$env_file"
  chmod 600 "$env_file"
}

rollback_runtime() {
  local status=$?
  trap - EXIT HUP INT TERM

  if [[ "$status" -ne 0 && "$revision_changed" == true ]]; then
    echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2
    restore_image_revision
    "$root/scripts/compose.sh" "$env_file" \
      up -d --no-deps --no-build --wait "${expected_services[@]}" || true

    edge_project=$(read_value EDGE_COMPOSE_PROJECT_NAME)
    docker compose \
      --project-name "$edge_project" \
      --project-directory "$root" \
      --env-file "$env_file" \
      --file "$root/compose.edge.yaml" \
      up -d --no-deps --no-build --wait edge || true

    curl --fail --silent --show-error --max-time 15 \
      "https://$expected_domain/healthz/ready" >/dev/null || true
    {
      printf 'status=runtime-rolled-back\n'
      printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
      printf 'database_note=migrations_may_have_applied_and_were_not_reversed\n'
    } >>"$rollback_manifest"
    echo "The Git checkout and any applied migrations were intentionally not reversed automatically." >&2
  fi

  exit "$status"
}
trap rollback_runtime EXIT HUP INT TERM

release_ref="refs/wnh/releases/$target_commit"
git -C "$root" fetch "$bundle" "HEAD:$release_ref"
[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || {
  echo "Fetched release ref does not match the approved commit." >&2
  exit 1
}
git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || {
  echo "Production updates must be a fast-forward from the deployed commit." >&2
  exit 1
}
if [[ "$branch" == "main" ]]; then
  git -C "$root" merge --ff-only "$release_ref"
else
  git -C "$root" checkout --detach "$release_ref"
fi

"$root/scripts/set-deployment-revision.sh" "$env_file"
revision_changed=true
"$root/scripts/validate-production-env.sh" "$env_file" "$expected_domain"
"$root/scripts/check-environment-readiness.sh" "$env_file" --require-release

"$root/scripts/deploy-up.sh" "$env_file"
"$root/scripts/edge-up.sh" "$env_file"
curl --fail --silent --show-error --max-time 30 \
  "https://$expected_domain/healthz/ready" >/dev/null
curl --fail --silent --show-error --max-time 30 \
  "https://$expected_domain/.well-known/assetlinks.json" >/dev/null

{
  printf 'status=success\n'
  printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} >>"$rollback_manifest"
revision_changed=false
trap - EXIT HUP INT TERM

printf 'Production release completed: %s\n' "$target_commit"
printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
printf 'Database backup: %s\n' "$backup"