379 lines
12 KiB
Bash
Executable File
379 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
action=${1:-}
|
|
root=${2:-/srv/who_need_help-production}
|
|
expected_domain=${3:-whoneedhelp.com}
|
|
manifest=${4:-}
|
|
|
|
usage() {
|
|
echo "Usage: $0 plan|apply /srv/who_need_help-production whoneedhelp.com ROLLBACK_MANIFEST" >&2
|
|
}
|
|
|
|
case "$action" in
|
|
plan | apply) ;;
|
|
*) usage; exit 2 ;;
|
|
esac
|
|
|
|
root=$(realpath --canonicalize-existing "$root")
|
|
if [[ "$root" != "/srv/who_need_help-production" ]]; then
|
|
echo "Refusing a production rollback outside /srv/who_need_help-production." >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ -z "$manifest" ]]; then
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
manifest=$(realpath --canonicalize-existing "$manifest")
|
|
case "$manifest" in
|
|
"$root"/output/releases/*/rollback-manifest.txt) ;;
|
|
*)
|
|
echo "Rollback manifest must be below $root/output/releases/." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
env_file="$root/.env"
|
|
if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then
|
|
echo "Production .env is missing or does not have mode 0600." >&2
|
|
exit 2
|
|
fi
|
|
if [[ "$(stat -c '%a' "$manifest")" != 600 ]]; then
|
|
echo "Rollback manifest must have mode 0600." >&2
|
|
exit 2
|
|
fi
|
|
|
|
read_unique() {
|
|
local file=$1 key=$2 count
|
|
count=$(awk -F= -v key="$key" '$1 == key {count++} END {print count + 0}' "$file")
|
|
if [[ "$count" -ne 1 ]]; then
|
|
echo "$key must occur exactly once in $file." >&2
|
|
exit 2
|
|
fi
|
|
awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1)}' "$file"
|
|
}
|
|
|
|
read_last() {
|
|
local file=$1 key=$2
|
|
awk -F= -v key="$key" '
|
|
$1 == key {value = substr($0, index($0, "=") + 1); found = 1}
|
|
END {if (!found) exit 1; print value}
|
|
' "$file"
|
|
}
|
|
|
|
require_commit() {
|
|
local value=$1 label=$2
|
|
[[ "$value" =~ ^[0-9a-f]{40}$ ]] || {
|
|
echo "$label is not a full Git commit." >&2
|
|
exit 2
|
|
}
|
|
}
|
|
|
|
require_image() {
|
|
local value=$1 prefix=$2 label=$3
|
|
[[ "$value" =~ ^who-need-help:${prefix}[A-Za-z0-9_.-]+$ ]] || {
|
|
echo "$label is not an expected immutable Who Need Help image tag." >&2
|
|
exit 2
|
|
}
|
|
}
|
|
|
|
deployment_environment=$(read_unique "$env_file" DEPLOYMENT_ENV)
|
|
compose_project=$(read_unique "$env_file" COMPOSE_PROJECT_NAME)
|
|
database_mode=$(read_unique "$env_file" DATABASE_MODE)
|
|
app_topology=$(read_unique "$env_file" APP_TOPOLOGY)
|
|
phx_host=$(read_unique "$env_file" PHX_HOST)
|
|
public_origin=$(read_unique "$env_file" WNH_BASE_URL)
|
|
edge_project=$(read_unique "$env_file" EDGE_COMPOSE_PROJECT_NAME)
|
|
|
|
[[ "$deployment_environment" == production ]] || {
|
|
echo "DEPLOYMENT_ENV is not production." >&2
|
|
exit 2
|
|
}
|
|
[[ "$compose_project" == who_need_help_production ]] || {
|
|
echo "Unexpected production Compose project." >&2
|
|
exit 2
|
|
}
|
|
[[ "$database_mode" == external ]] || {
|
|
echo "The verified production rollback workflow expects DATABASE_MODE=external." >&2
|
|
exit 2
|
|
}
|
|
[[ "$phx_host" == "$expected_domain" &&
|
|
"$public_origin" == "https://$expected_domain" ]] || {
|
|
echo "Production origin does not match the expected domain." >&2
|
|
exit 2
|
|
}
|
|
[[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || {
|
|
echo "Production checkout has tracked modifications." >&2
|
|
exit 2
|
|
}
|
|
|
|
"$root/scripts/validate-production-env.sh" "$env_file" "$expected_domain" >/dev/null
|
|
"$root/scripts/compose.sh" "$env_file" config --quiet
|
|
|
|
previous_commit=$(read_unique "$manifest" previous_commit)
|
|
target_commit=$(read_unique "$manifest" target_commit)
|
|
backup=$(read_unique "$manifest" database_backup)
|
|
release_status=$(read_last "$manifest" status)
|
|
migration_policy=$(read_unique "$manifest" migration_policy)
|
|
require_commit "$previous_commit" previous_commit
|
|
require_commit "$target_commit" target_commit
|
|
|
|
[[ "$release_status" == success ]] || {
|
|
echo "Only a manifest from a successful release can drive a manual rollback." >&2
|
|
exit 2
|
|
}
|
|
|
|
case "$migration_policy" in
|
|
application_safe) ;;
|
|
forward_only)
|
|
echo "This release is marked forward_only; automatic old-image rollback is blocked." >&2
|
|
echo "Inspect the exact database migration state and use a forward repair." >&2
|
|
exit 2
|
|
;;
|
|
*)
|
|
echo "The rollback manifest has an invalid migration policy." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
current_commit=$(git -C "$root" rev-parse --verify HEAD)
|
|
[[ "$current_commit" == "$target_commit" ]] || {
|
|
echo "The manifest target is not the currently checked-out production commit." >&2
|
|
exit 2
|
|
}
|
|
git -C "$root" cat-file -e "$previous_commit^{commit}"
|
|
git -C "$root" merge-base --is-ancestor "$previous_commit" "$target_commit" || {
|
|
echo "The manifest does not describe a forward production release." >&2
|
|
exit 2
|
|
}
|
|
|
|
previous_app_image=$(read_unique "$manifest" APP_IMAGE)
|
|
previous_socket_image=$(read_unique "$manifest" SOCKET_PROXY_IMAGE)
|
|
previous_postgis_image=$(read_unique "$manifest" POSTGIS_IMAGE)
|
|
previous_caddy_image=$(read_unique "$manifest" CADDY_IMAGE)
|
|
require_image "$previous_app_image" production- APP_IMAGE
|
|
require_image "$previous_socket_image" socket-proxy-production- SOCKET_PROXY_IMAGE
|
|
require_image "$previous_postgis_image" postgis-production- POSTGIS_IMAGE
|
|
require_image "$previous_caddy_image" caddy-production- CADDY_IMAGE
|
|
|
|
target_short=${target_commit:0:12}
|
|
current_app_image=$(read_unique "$env_file" APP_IMAGE)
|
|
current_socket_image=$(read_unique "$env_file" SOCKET_PROXY_IMAGE)
|
|
current_postgis_image=$(read_unique "$env_file" POSTGIS_IMAGE)
|
|
current_caddy_image=$(read_unique "$env_file" CADDY_IMAGE)
|
|
|
|
[[ "$current_app_image" == "who-need-help:production-$target_short" &&
|
|
"$current_socket_image" == "who-need-help:socket-proxy-production-$target_short" &&
|
|
"$current_postgis_image" == "who-need-help:postgis-production-$target_short" &&
|
|
"$current_caddy_image" == "who-need-help:caddy-production-$target_short" ]] || {
|
|
echo "Current production image selection does not match the manifest target commit." >&2
|
|
exit 2
|
|
}
|
|
|
|
for image in "$previous_app_image" "$previous_caddy_image"; do
|
|
docker image inspect "$image" >/dev/null
|
|
done
|
|
|
|
backup=$(realpath --canonicalize-existing "$backup")
|
|
case "$backup" in
|
|
"$root"/output/backups/production/*.dump) ;;
|
|
*)
|
|
echo "Manifest backup is outside the production backup directory." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
for required_file in "$backup" "$backup.sha256" "$backup.metadata"; do
|
|
[[ -f "$required_file" ]] || {
|
|
echo "Required rollback evidence is missing: $required_file" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
(
|
|
cd "$(dirname -- "$backup")"
|
|
sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
|
|
)
|
|
pg_restore --list "$backup" >/dev/null
|
|
|
|
case "$app_topology" in
|
|
compact) app_services=(app) ;;
|
|
split) app_services=(web worker) ;;
|
|
*) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;;
|
|
esac
|
|
|
|
check_application() {
|
|
local expected_image=$1 service container state health image
|
|
for service in "${app_services[@]}"; do
|
|
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
|
|
[[ ${#containers[@]} -gt 0 ]] || {
|
|
echo "Production service is not running: $service" >&2
|
|
return 1
|
|
}
|
|
for container in "${containers[@]}"; do
|
|
state=$(docker inspect --format '{{.State.Status}}' "$container")
|
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
|
image=$(docker inspect --format '{{.Config.Image}}' "$container")
|
|
[[ "$state" == running && "$health" == healthy && "$image" == "$expected_image" ]] || {
|
|
echo "Production service does not match the expected healthy image: $service" >&2
|
|
return 1
|
|
}
|
|
done
|
|
done
|
|
}
|
|
|
|
edge_compose=(
|
|
docker compose
|
|
--project-name "$edge_project"
|
|
--project-directory "$root"
|
|
--env-file "$env_file"
|
|
--file "$root/compose.edge.yaml"
|
|
)
|
|
|
|
check_edge() {
|
|
local expected_image=$1 container state health image
|
|
mapfile -t containers < <("${edge_compose[@]}" ps -q edge)
|
|
[[ ${#containers[@]} -gt 0 ]] || {
|
|
echo "Production edge service is not running." >&2
|
|
return 1
|
|
}
|
|
for container in "${containers[@]}"; do
|
|
state=$(docker inspect --format '{{.State.Status}}' "$container")
|
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
|
image=$(docker inspect --format '{{.Config.Image}}' "$container")
|
|
[[ "$state" == running && "$health" == healthy && "$image" == "$expected_image" ]] || {
|
|
echo "Production edge does not match the expected healthy image." >&2
|
|
return 1
|
|
}
|
|
done
|
|
}
|
|
|
|
check_application "$current_app_image"
|
|
check_edge "$current_caddy_image"
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null
|
|
|
|
confirmation="$expected_domain:$target_commit:$previous_commit"
|
|
printf 'Production checkout: %s\n' "$root"
|
|
printf 'Current source commit (unchanged by rollback): %s\n' "$target_commit"
|
|
printf 'Application image rollback commit: %s\n' "$previous_commit"
|
|
printf 'Compose project: %s\n' "$compose_project"
|
|
printf 'Topology: %s\n' "$app_topology"
|
|
printf 'Database mode: %s (no restore or migration reversal)\n' "$database_mode"
|
|
printf 'Rollback manifest: %s\n' "$manifest"
|
|
printf 'Verified backup evidence: %s\n' "$backup"
|
|
printf 'Exact confirmation: %s\n' "$confirmation"
|
|
echo "Scope: update four image selectors in production .env; recreate only application and edge containers."
|
|
echo "Excluded: Git checkout, database, migrations, test deployment, public Git, and Devpost."
|
|
|
|
if [[ "$action" == plan ]]; then
|
|
echo "Read-only production application rollback scope check passed."
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "${WNH_PRODUCTION_ROLLBACK_CONFIRM:-}" != "$confirmation" ]]; then
|
|
echo "Set WNH_PRODUCTION_ROLLBACK_CONFIRM=$confirmation for the approved rollback." >&2
|
|
exit 2
|
|
fi
|
|
|
|
update_images() {
|
|
local app_image=$1 socket_image=$2 postgis_image=$3 caddy_image=$4 temporary
|
|
temporary=$(mktemp "$root/.env.image-selection.XXXXXX")
|
|
chmod 600 "$temporary"
|
|
|
|
APP_IMAGE_VALUE=$app_image \
|
|
SOCKET_PROXY_IMAGE_VALUE=$socket_image \
|
|
POSTGIS_IMAGE_VALUE=$postgis_image \
|
|
CADDY_IMAGE_VALUE=$caddy_image \
|
|
awk '
|
|
BEGIN {
|
|
replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
|
|
replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
|
|
replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
|
|
replacement["CADDY_IMAGE"] = ENVIRON["CADDY_IMAGE_VALUE"]
|
|
}
|
|
{
|
|
separator = index($0, "=")
|
|
key = separator > 1 ? substr($0, 1, separator - 1) : ""
|
|
if (key in replacement) {
|
|
seen[key]++
|
|
print key "=" replacement[key]
|
|
} else {
|
|
print
|
|
}
|
|
}
|
|
END {
|
|
for (key in replacement) {
|
|
if (seen[key] != 1) exit 1
|
|
}
|
|
}
|
|
' "$env_file" >"$temporary" || {
|
|
rm -f "$temporary"
|
|
return 1
|
|
}
|
|
|
|
mv "$temporary" "$env_file"
|
|
chmod 600 "$env_file"
|
|
}
|
|
|
|
runtime_changed=false
|
|
recover_current_runtime() {
|
|
local status=$?
|
|
trap - EXIT HUP INT TERM
|
|
if [[ "$status" -ne 0 && "$runtime_changed" == true ]]; then
|
|
echo "Rollback failed; restoring the pre-rollback image selection." >&2
|
|
update_images \
|
|
"$current_app_image" \
|
|
"$current_socket_image" \
|
|
"$current_postgis_image" \
|
|
"$current_caddy_image" || true
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
up -d --no-deps --no-build --wait "${app_services[@]}" || true
|
|
"${edge_compose[@]}" \
|
|
up -d --no-deps --no-build --wait edge || true
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null || true
|
|
fi
|
|
exit "$status"
|
|
}
|
|
trap recover_current_runtime EXIT HUP INT TERM
|
|
|
|
update_images \
|
|
"$previous_app_image" \
|
|
"$previous_socket_image" \
|
|
"$previous_postgis_image" \
|
|
"$previous_caddy_image"
|
|
runtime_changed=true
|
|
|
|
"$root/scripts/compose.sh" "$env_file" \
|
|
up -d --no-deps --no-build --wait "${app_services[@]}"
|
|
"${edge_compose[@]}" \
|
|
up -d --no-deps --no-build --wait edge
|
|
|
|
check_application "$previous_app_image"
|
|
check_edge "$previous_caddy_image"
|
|
curl --fail --silent --show-error --max-time 30 \
|
|
"https://$expected_domain/healthz/ready" >/dev/null
|
|
curl --fail --silent --show-error --max-time 30 \
|
|
"https://$expected_domain/.well-known/assetlinks.json" >/dev/null
|
|
|
|
audit_file="$(dirname -- "$manifest")/application-rollback-$(date -u +%Y%m%dT%H%M%SZ).txt"
|
|
{
|
|
printf 'source_manifest=%s\n' "$manifest"
|
|
printf 'source_commit_retained=%s\n' "$target_commit"
|
|
printf 'application_images_restored_from_commit=%s\n' "$previous_commit"
|
|
printf 'database_action=none\n'
|
|
printf 'migration_action=none\n'
|
|
printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'status=success\n'
|
|
} >"$audit_file"
|
|
chmod 600 "$audit_file"
|
|
|
|
runtime_changed=false
|
|
trap - EXIT HUP INT TERM
|
|
|
|
printf 'Production application images rolled back to release %s.\n' "$previous_commit"
|
|
printf 'Production source remains at %s.\n' "$target_commit"
|
|
printf 'Rollback audit: %s\n' "$audit_file"
|