diff --git a/docs/operations.md b/docs/operations.md index 3f656bd..bcd1a79 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -924,13 +924,33 @@ The apply path refuses tracked local or remote modifications. It then: application and edge, and verifies the public readiness and Android App Links endpoints. -If application startup fails after the new image tags are selected, the script -restores the previous immutable application and Caddy image tags and attempts -to recover public readiness. It deliberately does not reverse Git source or -Ecto migrations automatically. The per-release rollback manifest and backup -paths are recorded below the production checkout's ignored `output/releases/`. -The copied backup is separate from the production host, but a long-term -encrypted off-site backup destination remains an operational requirement. +Every newly added migration must have one reviewed entry in +`priv/repo/migration_application_compatibility.tsv`. `application_safe` means +the previously deployed application can run against the resulting schema; +`forward_only` means that it cannot be guaranteed. The plan prints the +aggregate policy. A forward-only apply requires a second exact confirmation: + +```bash +WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=whoneedhelp.com:FULL_COMMIT:forward-only \ + WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \ + ./scripts/production-release.sh apply whoneedhelp +``` + +For a forward-only release, all candidate images are built first, the old +application is stopped before migration begins, and the target application is +started only after the migration runner succeeds. If anything fails after the +migration begins, the release deliberately leaves the old application stopped +and records that boundary in the release manifest. Restarting an older image +against a potentially incompatible schema is never automatic. + +For an `application_safe` release, an application startup failure restores the +previous immutable application and Caddy image tags and attempts to recover +public readiness. A `forward_only` release never starts the old application +after migration begins. Neither path reverses Git source or Ecto migrations +automatically. The per-release rollback manifest and backup paths are recorded +below the production checkout's ignored `output/releases/`. The copied backup +is separate from the production host, but a long-term encrypted off-site +backup destination remains an operational requirement. ## Rollback boundary @@ -950,6 +970,11 @@ rollback plan: whoneedhelp ``` +The rollback plan refuses manifests marked `forward_only`. Such releases must +be repaired forward after inspecting the exact migration state; an application +image rollback is available only when the manifest records +`migration_policy=application_safe`. + The plan requires the manifest target to be the currently checked-out production commit, verifies the previous immutable application and edge images still exist, checks the pre-release backup catalog and checksum, and prints the diff --git a/priv/repo/migration_application_compatibility.tsv b/priv/repo/migration_application_compatibility.tsv new file mode 100644 index 0000000..4e90270 --- /dev/null +++ b/priv/repo/migration_application_compatibility.tsv @@ -0,0 +1,5 @@ +# migration_version application_rollback_policy reason +20260722110837 forward_only hidden requests can store no coordinates and old code requires a point +20260722123052 application_safe additive partial geography index +20260722190604 forward_only assignment history permits rows the old single-assignment model cannot interpret safely +20260723015032 application_safe additive request-helper lookup index diff --git a/scripts/production-release-drill.sh b/scripts/production-release-drill.sh new file mode 100755 index 0000000..f44043a --- /dev/null +++ b/scripts/production-release-drill.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd" +run_dir=$(mktemp -d "$ROOT/output/release-drill.XXXXXX") +fixture="$run_dir/production" +mock_bin="$run_dir/mock-bin" +remote_root=/srv/who_need_help-production +current_commit=1111111111111111111111111111111111111111 +target_commit=2222222222222222222222222222222222222222 +release_confirmation="whoneedhelp.com:$target_commit" +forward_confirmation="whoneedhelp.com:$target_commit:forward-only" + +cleanup() { + trap - EXIT HUP INT TERM + find "$run_dir" -xdev -depth -delete 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +install -d -m 700 \ + "$fixture/.git" \ + "$fixture/scripts" \ + "$fixture/output/releases/incoming" \ + "$fixture/output/backups/production" \ + "$mock_bin" + +write_old_env() { + install -m 600 /dev/null "$fixture/.env" + printf '%s\n' \ + 'DEPLOYMENT_ENV=production' \ + 'COMPOSE_PROJECT_NAME=who_need_help_production' \ + 'DATABASE_MODE=external' \ + 'APP_TOPOLOGY=compact' \ + 'PHX_HOST=whoneedhelp.com' \ + 'WNH_BASE_URL=https://whoneedhelp.com' \ + 'EDGE_COMPOSE_PROJECT_NAME=who_need_help_edge' \ + "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \ + "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${current_commit:0:12}" \ + "POSTGIS_IMAGE=who-need-help:postgis-production-${current_commit:0:12}" \ + "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ + >"$fixture/.env" +} +write_old_env + +bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle" +printf 'isolated release drill bundle\n' >"$bundle" +bundle_hash=$(sha256sum "$bundle" | awk '{print $1}') +printf '%s %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256" +backup="$fixture/output/backups/production/pre-release.dump" +printf 'isolated release drill backup\n' >"$backup" +backup_hash=$(sha256sum "$backup" | awk '{print $1}') +printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256" +printf 'environment=production\n' >"$backup.metadata" +chmod 600 "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata" + +for script in validate-production-env.sh check-environment-readiness.sh \ + check-database.sh verify-realtime-cluster.sh verify-beam-runtime.sh; do + install -m 755 /dev/null "$fixture/scripts/$script" + printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/$script" +done + +install -m 755 /dev/null "$fixture/scripts/set-deployment-revision.sh" +cat >"$fixture/scripts/set-deployment-revision.sh" <<'EOF' +#!/bin/sh +set -eu +env_file=$1 +sed -i \ + -e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ + -e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ + -e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ + -e "s|^CADDY_IMAGE=.*|CADDY_IMAGE=who-need-help:caddy-production-${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}|" \ + "$env_file" +EOF + +install -m 755 /dev/null "$fixture/scripts/compose.sh" +cat >"$fixture/scripts/compose.sh" <<'EOF' +#!/bin/sh +set -eu +env_file=$1 +shift +case "$*" in + 'config --quiet') exit 0 ;; + 'ps -q app') printf 'app-1\n'; exit 0 ;; + 'build migrate') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + 'stop app') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + 'run --rm --no-deps migrate') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + 'up -d --no-deps --no-build --wait app') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + if [ "${MOCK_FAIL_APP_UP:-}" = once ] && + [ ! -e "$MOCK_FAIL_APP_MARKER" ]; then + : >"$MOCK_FAIL_APP_MARKER" + exit 23 + fi + exit 0 + ;; +esac +printf 'Unexpected compose invocation: %s\n' "$*" >&2 +exit 1 +EOF + +printf '%s\n' "$current_commit" >"$fixture/git-state" +install -m 755 /dev/null "$mock_bin/git" +cat >"$mock_bin/git" <<'EOF' +#!/bin/sh +set -eu +case " $* " in + *' symbolic-ref --quiet --short HEAD '*) exit 1 ;; + *' status --porcelain --untracked-files=no '*) exit 0 ;; + *' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;; + *' rev-parse refs/wnh/releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; + *' bundle verify '*) exit 0 ;; + *' bundle list-heads '*) printf '%s HEAD\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; + *' fetch '*) + printf 'git:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + *' merge-base --is-ancestor '*) exit 0 ;; + *' show refs/wnh/releases/'*':scripts/release-migration-policy.sh '*) + cat <<'POLICY' +#!/bin/sh +set -eu +printf 'migration_policy=%s\n' "$MOCK_MIGRATION_POLICY" +printf 'migration_versions=20260101000000:%s\n' "$MOCK_MIGRATION_POLICY" +printf 'migration_count=1\n' +POLICY + exit 0 + ;; + *' checkout --detach refs/wnh/releases/'*) + printf '%s\n' "$MOCK_TARGET_COMMIT" >"$MOCK_GIT_STATE" + exit 0 + ;; +esac +printf 'Unexpected git invocation: %s\n' "$*" >&2 +exit 1 +EOF + +install -m 755 /dev/null "$mock_bin/docker" +cat >"$mock_bin/docker" <<'EOF' +#!/bin/sh +set -eu +if [ "$1" = inspect ]; then + case "$3" in + '{{.State.Status}}') printf 'running\n' ;; + '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;; + *) exit 1 ;; + esac + exit 0 +fi +if [ "$1" = compose ]; then + case " $* " in + *' build edge '*) + printf 'edge:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + *' up -d --no-deps --no-build --wait edge '*) + printf 'edge:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + esac +fi +printf 'Unexpected docker invocation: %s\n' "$*" >&2 +exit 1 +EOF + +for command in curl pg_restore; do + install -m 755 /dev/null "$mock_bin/$command" + printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" +done + +touch "$fixture/mock-commands.log" +chmod 600 "$fixture/mock-commands.log" "$fixture/git-state" + +container_env=( + --env "MOCK_TARGET_COMMIT=$target_commit" + --env "MOCK_GIT_STATE=$remote_root/git-state" + --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" + --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +) +container_mounts=( + --volume "$fixture:$remote_root" + --volume "$mock_bin:/mock-bin:ro" + --volume "$ROOT/scripts/production-release-remote.sh:/runner/production-release-remote.sh:ro" +) + +run_release() { + local policy=$1 + shift + docker run --rm \ + --network none \ + --user "$(id -u):$(id -g)" \ + --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,noexec \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --env "MOCK_MIGRATION_POLICY=$policy" \ + --env "WNH_PRODUCTION_RELEASE_CONFIRM=$release_confirmation" \ + --env "WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation" \ + "$@" \ + "${container_env[@]}" \ + "${container_mounts[@]}" \ + "$BASE_IMAGE" \ + bash /runner/production-release-remote.sh \ + apply "$remote_root" whoneedhelp.com \ + "$remote_root/output/releases/incoming/$(basename -- "$bundle")" \ + "$target_commit" \ + "$remote_root/output/backups/production/$(basename -- "$backup")" \ + "$policy" +} + +run_release forward_only >"$run_dir/forward-success.out" +grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ + "$fixture/.env" >/dev/null +grep -F 'compose:build migrate' "$fixture/mock-commands.log" >/dev/null +grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null +grep -F 'compose:run --rm --no-deps migrate' \ + "$fixture/mock-commands.log" >/dev/null +grep -F 'compose:up -d --no-deps --no-build --wait app' \ + "$fixture/mock-commands.log" >/dev/null +grep -R -F 'migration_policy=forward_only' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null +grep -R -F 'status=success' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null + +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +rm -f "$fixture/app-up-failed" +set +e +run_release forward_only \ + --env MOCK_FAIL_APP_UP=once \ + --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ + >"$run_dir/forward-failure.out" 2>&1 +forward_failure_status=$? +set -e +if [[ "$forward_failure_status" -eq 0 ]]; then + echo "Forward-only release drill did not surface the injected startup failure." >&2 + exit 1 +fi +grep -F 'previous application will not be restarted' \ + "$run_dir/forward-failure.out" >/dev/null +grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ + "$fixture/.env" >/dev/null +test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \ + "$fixture/mock-commands.log")" = 1 +grep -R -F 'status=forward-only-release-failed' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null + +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +rm -f "$fixture/app-up-failed" +set +e +run_release application_safe \ + --env MOCK_FAIL_APP_UP=once \ + --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ + >"$run_dir/safe-failure.out" 2>&1 +safe_failure_status=$? +set -e +if [[ "$safe_failure_status" -eq 0 ]]; then + echo "Application-safe release drill did not surface the injected failure." >&2 + exit 1 +fi +grep -F 'restoring the previous immutable image tags' \ + "$run_dir/safe-failure.out" >/dev/null +grep -Fx "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \ + "$fixture/.env" >/dev/null +test "$(grep -Fc 'compose:up -d --no-deps --no-build --wait app' \ + "$fixture/mock-commands.log")" = 2 + +echo "Isolated production release success/forward-only/safe-recovery drill passed." diff --git a/scripts/production-release-remote.sh b/scripts/production-release-remote.sh index 3184ed1..265b49b 100755 --- a/scripts/production-release-remote.sh +++ b/scripts/production-release-remote.sh @@ -8,10 +8,11 @@ expected_domain=${3:-whoneedhelp.com} bundle=${4:-} target_commit=${5:-} backup=${6:-} +expected_migration_policy=${7:-} usage() { echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2 - echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP" >&2 + echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY" >&2 } case "$action" in @@ -121,7 +122,8 @@ if [[ "$action" == "plan" ]]; then exit 0 fi -if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ]]; then +if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" || + -z "$expected_migration_policy" ]]; then usage exit 2 fi @@ -156,7 +158,44 @@ bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {pr exit 2 } -release_id="$(date -u +%Y%m%dT%H%M%SZ)-${target_commit:0:12}" +release_ref="refs/wnh/releases/$target_commit" +git -C "$root" fetch "$bundle" "HEAD:$release_ref" +[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || { + echo "Fetched release ref does not match the approved commit." >&2 + exit 1 +} +git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { + echo "Production updates must be a fast-forward from the deployed commit." >&2 + exit 1 +} + +policy_script=$(mktemp) +trap 'rm -f "$policy_script"' EXIT HUP INT TERM +git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script" +chmod 600 "$policy_script" +migration_policy_output=$( + bash "$policy_script" "$current_commit" "$target_commit" "$root" +) +rm -f "$policy_script" +trap - EXIT HUP INT TERM +printf '%s\n' "$migration_policy_output" +migration_policy=$( + awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output" +) +[[ "$migration_policy" == "$expected_migration_policy" ]] || { + echo "Remote migration policy does not match the locally approved policy." >&2 + exit 2 +} + +if [[ "$migration_policy" == "forward_only" ]]; then + forward_confirmation="$expected_domain:$target_commit:forward-only" + [[ "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || { + echo "Set WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation for this schema boundary." >&2 + exit 2 + } +fi + +release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}" release_dir="$root/output/releases/$release_id" mkdir -p "$release_dir" chmod 700 "$root/output" "$root/output/releases" "$release_dir" @@ -169,6 +208,9 @@ rollback_manifest="$release_dir/rollback-manifest.txt" printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)" printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)" printf 'CADDY_IMAGE=%s\n' "$(read_value CADDY_IMAGE)" + printf 'migration_policy=%s\n' "$migration_policy" + printf 'migration_details=%s\n' \ + "$(awk -F= '$1 == "migration_versions" {print $2}' <<<"$migration_policy_output")" printf 'database_backup=%s\n' "$backup" printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'status=started\n' @@ -176,6 +218,7 @@ rollback_manifest="$release_dir/rollback-manifest.txt" chmod 600 "$rollback_manifest" revision_changed=false +migration_started=false restore_image_revision() { local temporary @@ -208,11 +251,21 @@ rollback_runtime() { local status=$? trap - EXIT HUP INT TERM - if [[ "$status" -ne 0 && "$revision_changed" == true ]]; then + if [[ "$status" -ne 0 && "$revision_changed" == true && + "$migration_policy" == "forward_only" && "$migration_started" == true ]]; then + { + printf 'status=forward-only-release-failed\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'automatic_application_rollback=blocked\n' + printf 'recovery_note=inspect migration state and repair the approved target release\n' + } >>"$rollback_manifest" + echo "Forward-only release failed after migration started." >&2 + echo "The previous application will not be restarted against a potentially incompatible schema." >&2 + elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2 restore_image_revision "$root/scripts/compose.sh" "$env_file" \ - up -d --no-deps --no-build --wait "${expected_services[@]}" || true + up -d --no-deps --no-build --wait "${runtime_services[@]}" || true edge_project=$(read_value EDGE_COMPOSE_PROJECT_NAME) docker compose \ @@ -234,18 +287,19 @@ rollback_runtime() { exit "$status" } +case "$app_topology" in + compact) + build_services=(migrate) + runtime_services=(app) + ;; + split) + build_services=(docker-api-proxy proxy migrate) + runtime_services=(docker-api-proxy proxy web worker) + ;; +esac + trap rollback_runtime EXIT HUP INT TERM -release_ref="refs/wnh/releases/$target_commit" -git -C "$root" fetch "$bundle" "HEAD:$release_ref" -[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || { - echo "Fetched release ref does not match the approved commit." >&2 - exit 1 -} -git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { - echo "Production updates must be a fast-forward from the deployed commit." >&2 - exit 1 -} if [[ "$branch" == "main" ]]; then git -C "$root" merge --ff-only "$release_ref" else @@ -257,8 +311,33 @@ revision_changed=true "$root/scripts/validate-production-env.sh" "$env_file" "$expected_domain" "$root/scripts/check-environment-readiness.sh" "$env_file" --require-release -"$root/scripts/deploy-up.sh" "$env_file" -"$root/scripts/edge-up.sh" "$env_file" +"$root/scripts/compose.sh" "$env_file" build "${build_services[@]}" +edge_project=$(read_value EDGE_COMPOSE_PROJECT_NAME) +edge_compose=( + docker compose + --project-name "$edge_project" + --project-directory "$root" + --env-file "$env_file" + --file "$root/compose.edge.yaml" +) +"${edge_compose[@]}" build edge + +if [[ "$migration_policy" == "forward_only" ]]; then + echo "Stopping the old application before the forward-only migration boundary." + "$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}" +fi + +migration_started=true +"$root/scripts/compose.sh" "$env_file" run --rm --no-deps migrate +"$root/scripts/check-database.sh" "$env_file" +"$root/scripts/compose.sh" "$env_file" \ + up -d --no-deps --no-build --wait "${runtime_services[@]}" +"${edge_compose[@]}" up -d --no-deps --no-build --wait edge + +COMPOSE_PROJECT_NAME="$compose_project" \ + "$root/scripts/verify-realtime-cluster.sh" compose +COMPOSE_PROJECT_NAME="$compose_project" \ + "$root/scripts/verify-beam-runtime.sh" compose curl --fail --silent --show-error --max-time 30 \ "https://$expected_domain/healthz/ready" >/dev/null curl --fail --silent --show-error --max-time 30 \ diff --git a/scripts/production-release.sh b/scripts/production-release.sh index 99bed50..8edd65b 100755 --- a/scripts/production-release.sh +++ b/scripts/production-release.sh @@ -43,6 +43,14 @@ else exit 2 fi +migration_policy_output=$( + "$ROOT/scripts/release-migration-policy.sh" "$remote_commit" "$local_commit" +) +printf '%s\n' "$migration_policy_output" +migration_policy=$( + awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output" +) + plan_failed=0 if ! ssh -o BatchMode=yes "$ssh_target" \ @@ -85,6 +93,18 @@ if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$confirmation" ]]; then exit 2 fi +if [[ "$migration_policy" == "forward_only" ]]; then + forward_confirmation="$expected_domain:$local_commit:forward-only" + if [[ "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" != "$forward_confirmation" ]]; then + echo "This release contains migrations that are not safe for an automatic old-image rollback." >&2 + echo "A failed deployment after migration starts will keep the old application stopped." >&2 + echo "Review the migration and recovery plan, then approve this exact boundary:" >&2 + echo "WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation \\" >&2 + echo " WNH_PRODUCTION_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2 + exit 2 + fi +fi + "$ROOT/scripts/prepare-production-release.sh" release_dir="$ROOT/output/releases/$local_commit" bundle="$release_dir/who_need_help-$local_commit.bundle" @@ -117,8 +137,11 @@ pg_restore --list "$local_backup_dir/$(basename -- "$remote_backup")" >/dev/null echo "Copied and independently verified the pre-release backup outside the production server." quoted_confirmation=$(printf '%q' "$confirmation") +quoted_forward_confirmation=$( + printf '%q' "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" +) ssh -o BatchMode=yes "$ssh_target" \ - "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup'" \ + "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy'" \ <"$ROOT/scripts/production-release-remote.sh" echo "Production release and public health verification completed." diff --git a/scripts/production-rollback-drill.sh b/scripts/production-rollback-drill.sh index 8ae9433..b7f9b0e 100755 --- a/scripts/production-rollback-drill.sh +++ b/scripts/production-rollback-drill.sh @@ -56,6 +56,7 @@ printf '%s\n' \ "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${previous_commit:0:12}" \ "POSTGIS_IMAGE=who-need-help:postgis-production-${previous_commit:0:12}" \ "CADDY_IMAGE=who-need-help:caddy-production-${previous_commit:0:12}" \ + 'migration_policy=application_safe' \ "database_backup=$remote_root/output/backups/production/pre-release.dump" \ 'status=started' \ 'status=success' \ @@ -207,6 +208,35 @@ find "$fixture/output/releases/release-1" \ grep -F "Production application images rolled back to release $previous_commit." \ "$run_dir/apply.out" >/dev/null +sed -i \ + 's/^migration_policy=application_safe$/migration_policy=forward_only/' \ + "$fixture/output/releases/release-1/rollback-manifest.txt" +set +e +docker run --rm \ + --network none \ + --user "$(id -u):$(id -g)" \ + --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,noexec \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + "${container_env[@]}" \ + "${container_mounts[@]}" \ + "$BASE_IMAGE" \ + bash /runner/production-rollback-remote.sh \ + plan "$remote_root" whoneedhelp.com "$manifest" \ + >"$run_dir/forward-only.out" 2>&1 +forward_only_status=$? +set -e +if [[ "$forward_only_status" -eq 0 ]]; then + echo "Rollback drill allowed a forward-only application rollback." >&2 + exit 1 +fi +grep -F 'automatic old-image rollback is blocked' \ + "$run_dir/forward-only.out" >/dev/null +sed -i \ + 's/^migration_policy=forward_only$/migration_policy=application_safe/' \ + "$fixture/output/releases/release-1/rollback-manifest.txt" + sed -i \ -e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:production-${target_commit:0:12}|" \ -e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${target_commit:0:12}|" \ diff --git a/scripts/production-rollback-remote.sh b/scripts/production-rollback-remote.sh index 18c0634..adab846 100755 --- a/scripts/production-rollback-remote.sh +++ b/scripts/production-rollback-remote.sh @@ -117,6 +117,7 @@ previous_commit=$(read_unique "$manifest" previous_commit) target_commit=$(read_unique "$manifest" target_commit) backup=$(read_unique "$manifest" database_backup) release_status=$(read_last "$manifest" status) +migration_policy=$(read_unique "$manifest" migration_policy) require_commit "$previous_commit" previous_commit require_commit "$target_commit" target_commit @@ -125,6 +126,19 @@ require_commit "$target_commit" target_commit exit 2 } +case "$migration_policy" in + application_safe) ;; + forward_only) + echo "This release is marked forward_only; automatic old-image rollback is blocked." >&2 + echo "Inspect the exact database migration state and use a forward repair." >&2 + exit 2 + ;; + *) + echo "The rollback manifest has an invalid migration policy." >&2 + exit 2 + ;; +esac + current_commit=$(git -C "$root" rev-parse --verify HEAD) [[ "$current_commit" == "$target_commit" ]] || { echo "The manifest target is not the currently checked-out production commit." >&2 diff --git a/scripts/quality.sh b/scripts/quality.sh index 39b498d..5ef5839 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -69,6 +69,12 @@ docker run --rm \ echo "Checking isolated production application rollback plan/apply" ./scripts/production-rollback-drill.sh +echo "Checking release migration compatibility policy" +./scripts/release-migration-policy-drill.sh + +echo "Checking isolated production release orchestration" +./scripts/production-release-drill.sh + echo "Checking Dockerfiles with Hadolint 2.14.0" for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \ Dockerfile.postgis Dockerfile.socket-proxy Dockerfile.traefik \ diff --git a/scripts/release-migration-policy-drill.sh b/scripts/release-migration-policy-drill.sh new file mode 100755 index 0000000..0b1e67f --- /dev/null +++ b/scripts/release-migration-policy-drill.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +fixture=$(mktemp -d "$ROOT/output/migration-policy-drill.XXXXXX") + +cleanup() { + trap - EXIT HUP INT TERM + find "$fixture" -xdev -depth -delete 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +git -C "$fixture" init --quiet +git -C "$fixture" config user.name "Who Need Help release drill" +git -C "$fixture" config user.email "release-drill@example.invalid" +install -d -m 700 "$fixture/priv/repo/migrations" +install -m 600 /dev/null \ + "$fixture/priv/repo/migration_application_compatibility.tsv" +git -C "$fixture" add . +git -C "$fixture" commit --quiet -m baseline +baseline=$(git -C "$fixture" rev-parse HEAD) + +printf '%s\n' 'defmodule SafeMigration do end' \ + >"$fixture/priv/repo/migrations/20260101000000_safe_migration.exs" +printf '%s\t%s\t%s\n' \ + 20260101000000 application_safe "additive test index" \ + >>"$fixture/priv/repo/migration_application_compatibility.tsv" +git -C "$fixture" add . +git -C "$fixture" commit --quiet -m safe +safe=$(git -C "$fixture" rev-parse HEAD) + +safe_output=$( + "$ROOT/scripts/release-migration-policy.sh" "$baseline" "$safe" "$fixture" +) +grep -Fx 'migration_policy=application_safe' <<<"$safe_output" >/dev/null +grep -Fx 'migration_versions=20260101000000:application_safe' \ + <<<"$safe_output" >/dev/null + +printf '%s\n' 'defmodule ForwardMigration do end' \ + >"$fixture/priv/repo/migrations/20260102000000_forward_migration.exs" +printf '%s\t%s\t%s\n' \ + 20260102000000 forward_only "old fixture cannot read the new representation" \ + >>"$fixture/priv/repo/migration_application_compatibility.tsv" +git -C "$fixture" add . +git -C "$fixture" commit --quiet -m forward +forward=$(git -C "$fixture" rev-parse HEAD) + +forward_output=$( + "$ROOT/scripts/release-migration-policy.sh" "$baseline" "$forward" "$fixture" +) +grep -Fx 'migration_policy=forward_only' <<<"$forward_output" >/dev/null +grep -Fx \ + 'migration_versions=20260101000000:application_safe,20260102000000:forward_only' \ + <<<"$forward_output" >/dev/null + +printf '%s\n' 'defmodule SafeMigrationChanged do end' \ + >"$fixture/priv/repo/migrations/20260101000000_safe_migration.exs" +git -C "$fixture" add . +git -C "$fixture" commit --quiet -m modified +modified=$(git -C "$fixture" rev-parse HEAD) + +if "$ROOT/scripts/release-migration-policy.sh" \ + "$safe" "$modified" "$fixture" >/dev/null 2>&1; then + echo "Migration policy accepted an edited migration file." >&2 + exit 1 +fi + +git -C "$fixture" checkout --quiet -b unknown "$safe" +printf '%s\n' 'defmodule UnknownMigration do end' \ + >"$fixture/priv/repo/migrations/20260103000000_unknown_migration.exs" +git -C "$fixture" add . +git -C "$fixture" commit --quiet -m unknown +unknown=$(git -C "$fixture" rev-parse HEAD) + +if "$ROOT/scripts/release-migration-policy.sh" \ + "$safe" "$unknown" "$fixture" >/dev/null 2>&1; then + echo "Migration policy accepted an unreviewed migration." >&2 + exit 1 +fi + +no_change_output=$( + "$ROOT/scripts/release-migration-policy.sh" "$safe" "$safe" "$fixture" +) +grep -Fx 'migration_policy=application_safe' <<<"$no_change_output" >/dev/null +grep -Fx 'migration_count=0' <<<"$no_change_output" >/dev/null + +echo "Isolated release migration policy drill passed." diff --git a/scripts/release-migration-policy.sh b/scripts/release-migration-policy.sh new file mode 100755 index 0000000..0db8df0 --- /dev/null +++ b/scripts/release-migration-policy.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=${3:-$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)} +previous_commit=${1:-} +target_commit=${2:-HEAD} +policy_path=priv/repo/migration_application_compatibility.tsv + +usage() { + echo "Usage: $0 PREVIOUS_COMMIT TARGET_COMMIT [REPOSITORY_ROOT]" >&2 +} + +if [[ -z "$previous_commit" || -z "$target_commit" ]]; then + usage + exit 2 +fi + +for commit in "$previous_commit" "$target_commit"; do + git -C "$ROOT" cat-file -e "$commit^{commit}" 2>/dev/null || { + echo "Commit is unavailable in the repository: $commit" >&2 + exit 2 + } +done + +git -C "$ROOT" merge-base --is-ancestor "$previous_commit" "$target_commit" || { + echo "The target commit is not a descendant of the previous commit." >&2 + exit 2 +} + +mapfile -t migration_changes < <( + git -C "$ROOT" diff --name-status "$previous_commit..$target_commit" -- \ + 'priv/repo/migrations/[0-9]*.exs' +) + +if [[ ${#migration_changes[@]} -eq 0 ]]; then + echo "migration_policy=application_safe" + echo "migration_versions=none" + echo "migration_count=0" + exit 0 +fi + +registry=$(git -C "$ROOT" show "$target_commit:$policy_path" 2>/dev/null) || { + echo "The target commit does not contain $policy_path." >&2 + exit 2 +} + +overall_policy=application_safe +versions=() + +for change in "${migration_changes[@]}"; do + status=${change%%$'\t'*} + path=${change#*$'\t'} + + if [[ "$status" != A ]]; then + echo "Applied migration files must not be modified, renamed, or deleted: $change" >&2 + exit 2 + fi + + filename=${path##*/} + version=${filename%%_*} + [[ "$version" =~ ^[0-9]{14}$ ]] || { + echo "Migration does not have a 14-digit Ecto version: $path" >&2 + exit 2 + } + + mapfile -t matches < <( + awk -F '\t' -v version="$version" ' + $0 !~ /^#/ && $1 == version {print $2 "\t" $3} + ' <<<"$registry" + ) + + if [[ ${#matches[@]} -ne 1 ]]; then + echo "Migration $version must have exactly one entry in $policy_path." >&2 + exit 2 + fi + + policy=${matches[0]%%$'\t'*} + reason=${matches[0]#*$'\t'} + + case "$policy" in + application_safe | forward_only) ;; + *) + echo "Migration $version has an invalid application rollback policy: $policy" >&2 + exit 2 + ;; + esac + + [[ -n "$reason" && "$reason" != "$policy" ]] || { + echo "Migration $version must document why its policy is correct." >&2 + exit 2 + } + + if [[ "$policy" == forward_only ]]; then + overall_policy=forward_only + fi + + versions+=("$version:$policy") +done + +printf 'migration_policy=%s\n' "$overall_policy" +printf 'migration_versions=%s\n' "$(IFS=,; echo "${versions[*]}")" +printf 'migration_count=%s\n' "${#versions[@]}"