who_need_help/scripts/test-release-remote.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

action=${1:-}
root=${2:-/srv/who_need_help-test}
expected_domain=${3:-test.whoneedhelp.com}
bundle=${4:-}
target_commit=${5:-}
backup=${6:-}
expected_migration_policy=${7:-}
image_archive=${8:-}
image_manifest=${9:-}

usage() {
  echo "Usage: $0 plan /srv/who_need_help-test test.whoneedhelp.com" >&2
  echo "       $0 apply /srv/who_need_help-test test.whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2
}

case "$action" in
  plan | apply) ;;
  *) usage; exit 2 ;;
esac

for command in curl docker git gzip jq pg_restore realpath sha256sum; do
  command -v "$command" >/dev/null 2>&1 || {
    echo "Required test release command is unavailable: $command" >&2
    exit 2
  }
done

root=$(realpath --canonicalize-existing "$root")
[[ "$root" == /srv/who_need_help-test ]] || {
  echo "Refusing a test release outside /srv/who_need_help-test." >&2
  exit 2
}

env_file="$root/.env"
[[ -f "$env_file" && "$(stat -c '%a' "$env_file")" == 600 ]] || {
  echo "Test .env is missing or does not have mode 0600." >&2
  exit 2
}

read_value() {
  local key=$1
  awk -v key="$key" '
    index($0, key "=") == 1 {
      print substr($0, length(key) + 2)
      found = 1
      exit
    }
    END { if (!found) exit 1 }
  ' "$env_file"
}

deployment_environment=$(read_value DEPLOYMENT_ENV)
compose_project=$(read_value COMPOSE_PROJECT_NAME)
database_mode=$(read_value DATABASE_MODE)
app_topology=$(read_value APP_TOPOLOGY)
phx_host=$(read_value PHX_HOST)
public_origin=$(read_value WNH_BASE_URL)
current_commit=$(git -C "$root" rev-parse --verify HEAD)

[[ "$deployment_environment" == test ]] || {
  echo "DEPLOYMENT_ENV is not test." >&2
  exit 2
}
[[ "$compose_project" == who_need_help_test ]] || {
  echo "Unexpected test Compose project." >&2
  exit 2
}
[[ "$database_mode" == container ]] || {
  echo "The verified test workflow expects DATABASE_MODE=container." >&2
  exit 2
}
[[ "$phx_host" == "$expected_domain" &&
  "$public_origin" == "https://$expected_domain" ]] || {
  echo "Test origin does not match the expected domain." >&2
  exit 2
}
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
  echo "Test checkout has uncommitted files." >&2
  exit 2
}

"$root/scripts/compose.sh" "$env_file" config --quiet

case "$app_topology" in
  compact)
    expected_services=(app)
    runtime_services=(app)
    ;;
  split)
    expected_services=(web worker)
    runtime_services=(docker-api-proxy proxy web worker)
    ;;
  *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;;
esac

for service in db "${expected_services[@]}"; do
  mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
  [[ ${#containers[@]} -gt 0 ]] || {
    echo "Test service is not running: $service" >&2
    exit 2
  }
  for container in "${containers[@]}"; do
    state=$(docker inspect --format '{{.State.Status}}' "$container")
    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
    [[ "$state" == running && "$health" == healthy ]] || {
      echo "Test container is not healthy: $service" >&2
      exit 2
    }
  done
done

curl --fail --silent --show-error --max-time 15 \
  "https://$expected_domain/healthz/ready" >/dev/null

printf 'Test checkout: %s\n' "$root"
printf 'Current commit: %s\n' "$current_commit"
printf 'Compose project: %s\n' "$compose_project"
printf 'Topology: %s\n' "$app_topology"
printf 'Database mode: %s\n' "$database_mode"
printf 'Public readiness: passed\n'
df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root"

if [[ "$action" == plan ]]; then
  echo "Read-only test release scope check passed."
  exit 0
fi

if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
  -z "$expected_migration_policy" || -z "$image_archive" ||
  -z "$image_manifest" ]]; then
  usage
  exit 2
fi

expected_confirmation="$expected_domain:$target_commit"
[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$expected_confirmation" ]] || {
  echo "Set WNH_TEST_RELEASE_CONFIRM=$expected_confirmation for the approved test release." >&2
  exit 2
}

for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
  "$image_archive" "$image_archive.sha256" "$image_manifest"; do
  [[ -f "$required_file" ]] || {
    echo "Required test release evidence is missing: $required_file" >&2
    exit 2
  }
done

(
  cd "$(dirname -- "$bundle")"
  sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null
)
(
  cd "$(dirname -- "$backup")"
  sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
)
pg_restore --list "$backup" >/dev/null
(
  cd "$(dirname -- "$image_archive")"
  sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null
)
gzip -t "$image_archive"
grep -Fx 'format=1' "$image_manifest" >/dev/null
grep -Fx 'deployment=test' "$image_manifest" >/dev/null
grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
git -C "$root" bundle verify "$bundle" >/dev/null

bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
[[ "$bundle_head" == "$target_commit" ]] || {
  echo "Bundle HEAD does not match the approved test commit." >&2
  exit 2
}

release_ref="refs/wnh/test-releases/$target_commit"
git -C "$root" fetch "$bundle" "HEAD:$release_ref"
[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || {
  echo "Fetched test release ref does not match the approved commit." >&2
  exit 1
}
git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || {
  echo "Test updates must be a fast-forward from the deployed commit." >&2
  exit 1
}

policy_script=$(mktemp)
trap 'rm -f "$policy_script"' EXIT HUP INT TERM
git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script"
chmod 700 "$policy_script"
migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root")
rm -f "$policy_script"
trap - EXIT HUP INT TERM
printf '%s\n' "$migration_policy_output"
migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output")
[[ "$migration_policy" == "$expected_migration_policy" ]] || {
  echo "Remote migration policy does not match the locally approved policy." >&2
  exit 2
}

if [[ "$migration_policy" == forward_only ]]; then
  forward_confirmation="$expected_domain:$target_commit:forward-only"
  [[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
    echo "Set WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation for this test schema boundary." >&2
    exit 2
  }
fi

release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}"
release_dir="$root/output/releases/$release_id"
mkdir -p "$release_dir"
chmod 700 "$root/output" "$root/output/releases" "$release_dir"
rollback_manifest="$release_dir/rollback-manifest.txt"
{
  printf 'previous_commit=%s\n' "$current_commit"
  printf 'target_commit=%s\n' "$target_commit"
  printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)"
  printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)"
  printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)"
  printf 'migration_policy=%s\n' "$migration_policy"
  printf 'database_backup=%s\n' "$backup"
  printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  printf 'status=started\n'
} >"$rollback_manifest"
chmod 600 "$rollback_manifest"

revision_changed=false
migration_started=false

restore_previous_revision() {
  local temporary
  temporary=$(mktemp "$root/.env.test-release-rollback.XXXXXX")
  chmod 600 "$temporary"
  APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
  SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
  POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
    awk '
      BEGIN {
        replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
        replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
        replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
      }
      {
        separator = index($0, "=")
        key = separator > 1 ? substr($0, 1, separator - 1) : ""
        print (key in replacement) ? key "=" replacement[key] : $0
      }
    ' "$env_file" >"$temporary"
  mv "$temporary" "$env_file"
  chmod 600 "$env_file"
  git -C "$root" checkout --detach "$current_commit" >/dev/null
}

rollback_runtime() {
  local status=$?
  trap - EXIT HUP INT TERM
  if [[ "$status" -ne 0 && "$revision_changed" == true &&
    "$migration_policy" == forward_only && "$migration_started" == true ]]; then
    {
      printf 'status=forward-only-release-failed\n'
      printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
      printf 'automatic_application_rollback=blocked\n'
    } >>"$rollback_manifest"
    echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2
  elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then
    echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2
    restore_previous_revision
    "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db || true
    "$root/scripts/compose.sh" "$env_file" \
      up -d --no-deps --no-build --force-recreate --wait \
      "${runtime_services[@]}" || true
    {
      printf 'status=runtime-rolled-back\n'
      printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
    } >>"$rollback_manifest"
  fi
  exit "$status"
}
trap rollback_runtime EXIT HUP INT TERM

gzip -dc "$image_archive" | docker load >/dev/null
git -C "$root" checkout --detach "$release_ref" >/dev/null
"$root/scripts/set-deployment-revision.sh" "$env_file"
revision_changed=true
"$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain"

expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)")
if [[ "$app_topology" == split ]]; then
  expected_images+=(
    "$(read_value SOCKET_PROXY_IMAGE)"
    "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')"
  )
fi
grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null
test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}"
for image in "${expected_images[@]}"; do
  expected_id=$(awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' "$image_manifest")
  [[ -n "$expected_id" ]] || {
    echo "Image manifest is missing the expected image: $image" >&2
    exit 2
  }
  [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || {
    echo "Loaded test image ID does not match the manifest: $image" >&2
    exit 2
  }
  [[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || {
    echo "Loaded test image is not linux/amd64: $image" >&2
    exit 2
  }
done

"$root/scripts/restore-drill-compose.sh" "$backup"

if [[ "$migration_policy" == forward_only ]]; then
  echo "Stopping the old test application before the forward-only migration boundary."
  "$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}"
fi

"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db

verify_service_image() {
  local service=$1 expected_image=$2 require_health=$3
  local expected_image_id container state health configured_image running_image_id

  expected_image_id=$(docker image inspect --format '{{.Id}}' "$expected_image")
  mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
  [[ ${#containers[@]} -gt 0 ]] || {
    echo "Candidate test service has no container: $service" >&2
    return 1
  }

  for container in "${containers[@]}"; do
    state=$(docker inspect --format '{{.State.Status}}' "$container")
    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
    configured_image=$(docker inspect --format '{{.Config.Image}}' "$container")
    running_image_id=$(docker inspect --format '{{.Image}}' "$container")

    [[ "$state" == running ]] || {
      echo "Candidate test container is not running: $service" >&2
      return 1
    }
    if [[ "$require_health" == true && "$health" != healthy ]]; then
      echo "Candidate test container is not healthy: $service" >&2
      return 1
    fi
    [[ "$configured_image" == "$expected_image" &&
      "$running_image_id" == "$expected_image_id" ]] || {
      echo "Candidate test service does not use its approved image: $service" >&2
      return 1
    }
  done
}

verify_service_image db "$(read_value POSTGIS_IMAGE)" true
migration_started=true
"$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate
"$root/scripts/check-database.sh" "$env_file" </dev/null
"$root/scripts/compose.sh" "$env_file" \
  up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}"

expected_app_image=$(read_value APP_IMAGE)
for service in "${expected_services[@]}"; do
  verify_service_image "$service" "$expected_app_image" true
done
if [[ "$app_topology" == split ]]; then
  verify_service_image docker-api-proxy "$(read_value SOCKET_PROXY_IMAGE)" false
  verify_service_image proxy \
    "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" \
    false
fi

COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-realtime-cluster.sh" compose
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-beam-runtime.sh" compose
curl --fail --silent --show-error --max-time 30 \
  "https://$expected_domain/healthz/ready" >/dev/null

{
  printf 'status=success\n'
  printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} >>"$rollback_manifest"
revision_changed=false
trap - EXIT HUP INT TERM

printf 'Test release completed: %s\n' "$target_commit"
printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
printf 'Database backup: %s\n' "$backup"