473 lines
16 KiB
Bash
Executable File
473 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
action=${1:-}
|
|
root=${2:-/srv/who_need_help-production}
|
|
expected_domain=${3:-whoneedhelp.com}
|
|
bundle=${4:-}
|
|
target_commit=${5:-}
|
|
backup=${6:-}
|
|
expected_migration_policy=${7:-}
|
|
image_archive=${8:-}
|
|
image_manifest=${9:-}
|
|
test_evidence=${10:-}
|
|
|
|
usage() {
|
|
echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2
|
|
echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST TEST_EVIDENCE" >&2
|
|
}
|
|
|
|
case "$action" in
|
|
plan | apply) ;;
|
|
*) usage; exit 2 ;;
|
|
esac
|
|
|
|
root=$(realpath --canonicalize-existing "$root")
|
|
if [[ "$root" != "/srv/who_need_help-production" ]]; then
|
|
echo "Refusing a production release outside /srv/who_need_help-production." >&2
|
|
exit 2
|
|
fi
|
|
|
|
for command in curl docker flock git gzip jq pg_restore sha256sum; do
|
|
command -v "$command" >/dev/null 2>&1 || {
|
|
echo "Required production command is unavailable: $command" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
env_file="$root/.env"
|
|
if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then
|
|
echo "Production .env is missing or does not have mode 0600." >&2
|
|
exit 2
|
|
fi
|
|
|
|
read_value() {
|
|
local key=$1
|
|
awk -v key="$key" '
|
|
index($0, key "=") == 1 {
|
|
value = substr($0, length(key) + 2)
|
|
count += 1
|
|
}
|
|
END {
|
|
if (count != 1) {
|
|
printf "Expected exactly one %s entry in the production environment; found %d.\n", key, count > "/dev/stderr"
|
|
exit 1
|
|
}
|
|
print value
|
|
}
|
|
' "$env_file"
|
|
}
|
|
|
|
require_manifest_value() {
|
|
local file=$1 key=$2 expected=$3
|
|
awk -v key="$key" -v expected="$expected" '
|
|
index($0, key "=") == 1 {
|
|
value = substr($0, length(key) + 2)
|
|
count += 1
|
|
}
|
|
END {
|
|
if (count != 1 || value != expected) {
|
|
printf "Expected exactly one %s=%s entry in %s.\n", key, expected, FILENAME > "/dev/stderr"
|
|
exit 1
|
|
}
|
|
}
|
|
' "$file"
|
|
}
|
|
|
|
critical_env_keys=(
|
|
DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY
|
|
PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE DATABASE_URL
|
|
)
|
|
for critical_key in "${critical_env_keys[@]}"; do
|
|
read_value "$critical_key" >/dev/null
|
|
done
|
|
|
|
deployment_environment=$(read_value DEPLOYMENT_ENV)
|
|
compose_project=$(read_value COMPOSE_PROJECT_NAME)
|
|
database_mode=$(read_value DATABASE_MODE)
|
|
app_topology=$(read_value APP_TOPOLOGY)
|
|
phx_host=$(read_value PHX_HOST)
|
|
public_origin=$(read_value WNH_BASE_URL)
|
|
branch=$(git -C "$root" symbolic-ref --quiet --short HEAD || true)
|
|
current_commit=$(git -C "$root" rev-parse --verify HEAD)
|
|
|
|
[[ "$deployment_environment" == "production" ]] || {
|
|
echo "DEPLOYMENT_ENV is not production." >&2
|
|
exit 2
|
|
}
|
|
[[ "$compose_project" == "who_need_help_production" ]] || {
|
|
echo "Unexpected production Compose project." >&2
|
|
exit 2
|
|
}
|
|
[[ "$database_mode" == "external" ]] || {
|
|
echo "The verified single-server production workflow expects DATABASE_MODE=external." >&2
|
|
exit 2
|
|
}
|
|
[[ "$phx_host" == "$expected_domain" &&
|
|
"$public_origin" == "https://$expected_domain" ]] || {
|
|
echo "Production origin does not match the expected domain." >&2
|
|
exit 2
|
|
}
|
|
[[ "$branch" == "main" || -z "$branch" ]] || {
|
|
echo "Production checkout must be on main or detached at the deployed commit." >&2
|
|
exit 2
|
|
}
|
|
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
|
|
echo "Production checkout has uncommitted files." >&2
|
|
exit 2
|
|
}
|
|
|
|
"$root/scripts/compose.sh" "$env_file" config --quiet
|
|
|
|
[[ "$app_topology" == compact ]] || {
|
|
echo "The verified single-server production release workflow requires APP_TOPOLOGY=compact." >&2
|
|
exit 2
|
|
}
|
|
expected_services=(app)
|
|
runtime_services=(app)
|
|
|
|
verify_service_image() {
|
|
local service=$1 expected_image=$2 expected_image_id=$3
|
|
local container configured_image running_image_id state health
|
|
|
|
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
|
|
[[ ${#containers[@]} -gt 0 ]] || {
|
|
echo "Production runtime service has no container: $service" >&2
|
|
return 1
|
|
}
|
|
|
|
for container in "${containers[@]}"; do
|
|
state=$(docker inspect --format '{{.State.Status}}' "$container")
|
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
|
configured_image=$(docker inspect --format '{{.Config.Image}}' "$container")
|
|
running_image_id=$(docker inspect --format '{{.Image}}' "$container")
|
|
[[ "$state" == "running" && "$health" == "healthy" ]] || {
|
|
echo "Production runtime container is not healthy: $service" >&2
|
|
return 1
|
|
}
|
|
[[ "$configured_image" == "$expected_image" &&
|
|
"$running_image_id" == "$expected_image_id" ]] || {
|
|
echo "Production runtime does not use its approved image: $service" >&2
|
|
return 1
|
|
}
|
|
done
|
|
}
|
|
|
|
current_app_image=$(read_value APP_IMAGE)
|
|
current_app_image_id=$(docker image inspect --format '{{.Id}}' "$current_app_image")
|
|
verify_service_image app "$current_app_image" "$current_app_image_id"
|
|
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null
|
|
|
|
printf 'Production checkout: %s\n' "$root"
|
|
printf 'Current commit: %s\n' "$current_commit"
|
|
printf 'Branch: %s\n' "${branch:-detached}"
|
|
printf 'Compose project: %s\n' "$compose_project"
|
|
printf 'Topology: %s\n' "$app_topology"
|
|
printf 'Database mode: %s\n' "$database_mode"
|
|
printf 'Public readiness: passed\n'
|
|
df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root"
|
|
|
|
if [[ "$action" == "plan" ]]; then
|
|
echo "Read-only production release scope check passed."
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
|
|
-z "$expected_migration_policy" || -z "$image_archive" ||
|
|
-z "$image_manifest" || -z "$test_evidence" ]]; then
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
expected_confirmation="$expected_domain:$target_commit"
|
|
if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$expected_confirmation" ]]; then
|
|
echo "Set WNH_PRODUCTION_RELEASE_CONFIRM=$expected_confirmation for the approved release." >&2
|
|
exit 2
|
|
fi
|
|
|
|
for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
|
|
"$backup.metadata" "$image_archive" "$image_archive.sha256" \
|
|
"$image_manifest" "$test_evidence"; do
|
|
[[ -f "$required_file" ]] || {
|
|
echo "Required release evidence is missing: $required_file" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
(
|
|
cd "$(dirname -- "$bundle")"
|
|
sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null
|
|
)
|
|
(
|
|
cd "$(dirname -- "$backup")"
|
|
sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
|
|
)
|
|
pg_restore --list "$backup" >/dev/null
|
|
(
|
|
cd "$(dirname -- "$image_archive")"
|
|
sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null
|
|
)
|
|
gzip -t "$image_archive"
|
|
grep -Fx 'format=1' "$image_manifest" >/dev/null
|
|
grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
|
|
grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
|
|
require_manifest_value "$test_evidence" format 1
|
|
require_manifest_value "$test_evidence" deployment test
|
|
require_manifest_value "$test_evidence" commit "$target_commit"
|
|
require_manifest_value "$test_evidence" domain test.whoneedhelp.com
|
|
require_manifest_value "$test_evidence" status success
|
|
require_manifest_value "$test_evidence" public_health passed
|
|
git -C "$root" bundle verify "$bundle" >/dev/null
|
|
|
|
exec {release_lock_fd}>"$root/.git/wnh-production-release.lock"
|
|
chmod 600 "$root/.git/wnh-production-release.lock"
|
|
flock -n "$release_lock_fd" || {
|
|
echo "Another production release already holds $root/.git/wnh-production-release.lock." >&2
|
|
exit 1
|
|
}
|
|
|
|
bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
|
|
[[ "$bundle_head" == "$target_commit" ]] || {
|
|
echo "Bundle HEAD does not match the approved target commit." >&2
|
|
exit 2
|
|
}
|
|
|
|
release_ref="refs/wnh/releases/$target_commit"
|
|
git -C "$root" fetch "$bundle" "HEAD:$release_ref"
|
|
[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || {
|
|
echo "Fetched release ref does not match the approved commit." >&2
|
|
exit 1
|
|
}
|
|
git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || {
|
|
echo "Production updates must be a fast-forward from the deployed commit." >&2
|
|
exit 1
|
|
}
|
|
|
|
policy_script=$(mktemp)
|
|
# Invoked by the EXIT/HUP/INT/TERM traps below.
|
|
# shellcheck disable=SC2329
|
|
cleanup_policy_script() {
|
|
local status=$?
|
|
trap - EXIT HUP INT TERM
|
|
rm -f "$policy_script"
|
|
exit "$status"
|
|
}
|
|
trap cleanup_policy_script EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script"
|
|
chmod 600 "$policy_script"
|
|
migration_policy_output=$(
|
|
bash "$policy_script" "$current_commit" "$target_commit" "$root"
|
|
)
|
|
rm -f "$policy_script"
|
|
unset -f cleanup_policy_script
|
|
trap - EXIT HUP INT TERM
|
|
printf '%s\n' "$migration_policy_output"
|
|
migration_policy=$(
|
|
awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output"
|
|
)
|
|
[[ "$migration_policy" == "$expected_migration_policy" ]] || {
|
|
echo "Remote migration policy does not match the locally approved policy." >&2
|
|
exit 2
|
|
}
|
|
|
|
if [[ "$migration_policy" == "forward_only" ]]; then
|
|
forward_confirmation="$expected_domain:$target_commit:forward-only"
|
|
[[ "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
|
|
echo "Set WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation for this schema boundary." >&2
|
|
exit 2
|
|
}
|
|
fi
|
|
|
|
release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}"
|
|
release_dir="$root/output/releases/$release_id"
|
|
[[ -d "$root/output/releases" ]] || {
|
|
echo "Production release output directory is missing: $root/output/releases" >&2
|
|
exit 2
|
|
}
|
|
install -d -m 700 "$release_dir"
|
|
rollback_manifest="$release_dir/rollback-manifest.txt"
|
|
|
|
{
|
|
printf 'previous_commit=%s\n' "$current_commit"
|
|
printf 'target_commit=%s\n' "$target_commit"
|
|
printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)"
|
|
printf 'APP_IMAGE_ID=%s\n' "$current_app_image_id"
|
|
printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)"
|
|
printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)"
|
|
printf 'migration_policy=%s\n' "$migration_policy"
|
|
printf 'migration_details=%s\n' \
|
|
"$(awk -F= '$1 == "migration_versions" {print $2}' <<<"$migration_policy_output")"
|
|
printf 'database_backup=%s\n' "$backup"
|
|
printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'status=started\n'
|
|
} >"$rollback_manifest"
|
|
chmod 600 "$rollback_manifest"
|
|
|
|
revision_changed=false
|
|
migration_started=false
|
|
|
|
restore_image_revision() {
|
|
local temporary
|
|
temporary=$(mktemp "$root/.env.release-rollback.XXXXXX")
|
|
chmod 600 "$temporary"
|
|
|
|
APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
|
SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
|
POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
|
awk '
|
|
BEGIN {
|
|
replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
|
|
replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
|
|
replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
|
|
}
|
|
{
|
|
separator = index($0, "=")
|
|
key = separator > 1 ? substr($0, 1, separator - 1) : ""
|
|
print (key in replacement) ? key "=" replacement[key] : $0
|
|
}
|
|
' "$env_file" >"$temporary"
|
|
|
|
mv "$temporary" "$env_file"
|
|
chmod 600 "$env_file"
|
|
git -C "$root" checkout --detach "$current_commit" >/dev/null
|
|
}
|
|
|
|
rollback_runtime() {
|
|
local status=$?
|
|
local rollback_ok=false
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
|
|
if [[ "$status" -ne 0 && "$revision_changed" == true &&
|
|
"$migration_policy" == "forward_only" && "$migration_started" == true ]]; then
|
|
{
|
|
printf 'status=forward-only-release-failed\n'
|
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'automatic_application_rollback=blocked\n'
|
|
printf 'recovery_note=inspect migration state and repair the approved target release\n'
|
|
} >>"$rollback_manifest"
|
|
echo "Forward-only release failed after migration started." >&2
|
|
echo "The previous application will not be restarted against a potentially incompatible schema." >&2
|
|
elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then
|
|
echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2
|
|
previous_app_image=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest")
|
|
previous_app_image_id=$(awk -F= '$1 == "APP_IMAGE_ID" {print substr($0,index($0,"=")+1)}' "$rollback_manifest")
|
|
|
|
if restore_image_revision &&
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
up -d --no-deps --no-build --force-recreate --wait \
|
|
"${runtime_services[@]}" &&
|
|
verify_service_image app "$previous_app_image" "$previous_app_image_id" &&
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null; then
|
|
rollback_ok=true
|
|
fi
|
|
|
|
if [[ "$rollback_ok" == true ]]; then
|
|
{
|
|
printf 'status=runtime-rolled-back\n'
|
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'database_note=migrations_may_have_applied_and_were_not_reversed\n'
|
|
} >>"$rollback_manifest"
|
|
else
|
|
{
|
|
printf 'status=rollback-failed\n'
|
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
} >>"$rollback_manifest"
|
|
echo "Automatic production rollback failed; inspect the rollback manifest and runtime before retrying." >&2
|
|
fi
|
|
fi
|
|
|
|
exit "$status"
|
|
}
|
|
trap rollback_runtime EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
if [[ "$branch" == "main" ]]; then
|
|
git -C "$root" merge --ff-only "$release_ref"
|
|
else
|
|
git -C "$root" checkout --detach "$release_ref"
|
|
fi
|
|
revision_changed=true
|
|
"$root/scripts/set-deployment-revision.sh" "$env_file" --allow-pre-play
|
|
"$root/scripts/validate-production-env.sh" \
|
|
"$env_file" "$expected_domain" --allow-pre-play
|
|
"$root/scripts/check-environment-readiness.sh" \
|
|
"$env_file" --require-server-release
|
|
|
|
expected_images=("$(read_value APP_IMAGE)")
|
|
declare -A approved_image_ids=()
|
|
|
|
grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null
|
|
test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}"
|
|
for image in "${expected_images[@]}"; do
|
|
awk -F'|' -v image="$image" '
|
|
$1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 }
|
|
END { if (!found) exit 1 }
|
|
' "$image_manifest"
|
|
done
|
|
|
|
gzip -dc "$image_archive" | docker load >/dev/null
|
|
for image in "${expected_images[@]}"; do
|
|
expected_id=$(
|
|
awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' \
|
|
"$image_manifest"
|
|
)
|
|
[[ -n "$expected_id" ]] || {
|
|
echo "Image manifest is missing the expected image ID: $image" >&2
|
|
exit 2
|
|
}
|
|
approved_image_ids["$image"]=$expected_id
|
|
[[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || {
|
|
echo "Loaded image ID does not match the signed manifest: $image" >&2
|
|
exit 2
|
|
}
|
|
[[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || {
|
|
echo "Loaded production image is not linux/amd64: $image" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
if [[ "$migration_policy" == "forward_only" ]]; then
|
|
echo "Stopping the old application before the forward-only migration boundary."
|
|
"$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}"
|
|
fi
|
|
|
|
migration_started=true
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
run --rm --no-deps --interactive=false migrate
|
|
"$root/scripts/check-database.sh" "$env_file" </dev/null
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}"
|
|
expected_app_image=$(read_value APP_IMAGE)
|
|
verify_service_image app "$expected_app_image" \
|
|
"${approved_image_ids[$expected_app_image]}"
|
|
|
|
COMPOSE_PROJECT_NAME="$compose_project" \
|
|
"$root/scripts/verify-realtime-cluster.sh" compose
|
|
COMPOSE_PROJECT_NAME="$compose_project" \
|
|
"$root/scripts/verify-beam-runtime.sh" compose
|
|
curl --fail --silent --show-error --max-time 30 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null
|
|
curl --fail --silent --show-error --max-time 30 \
|
|
"https://$expected_domain/.well-known/assetlinks.json" >/dev/null
|
|
|
|
{
|
|
printf 'status=success\n'
|
|
printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
} >>"$rollback_manifest"
|
|
revision_changed=false
|
|
trap - EXIT HUP INT TERM
|
|
|
|
printf 'Production release completed: %s\n' "$target_commit"
|
|
printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
|
|
printf 'Database backup: %s\n' "$backup"
|