diff --git a/docs/operations.md b/docs/operations.md index 9c27d8f..56d7c4f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -361,14 +361,32 @@ TEST_GOOGLE_OAUTH_AUTHORIZED_PARTY_IDS=YOUR_TEST_ANDROID_CLIENT_ID \ ./scripts/deploy-up.sh .env ``` -For later test updates, check out the desired clean revision and update only -the image tags. Existing deployment secrets remain unchanged: +For later public test updates, release the exact clean revision with immutable +images built on the operator workstation. The test release workflow performs +a read-only scope plan, builds and verifies `linux/amd64` images outside the +4-GiB server, creates and copies an independently verified database backup, +runs a restore-and-migrate drill, and starts Compose with `--no-build`: ```bash -./scripts/set-deployment-revision.sh .env -./scripts/deploy-up.sh .env +./scripts/test-release.sh plan whoneedhelp +./scripts/test-release.sh prepare whoneedhelp + +# Make the selected commit available as origin/main before apply. The apply +# gate verifies that origin/main equals the exact local SHA. +candidate=$(git rev-parse HEAD) +WNH_TEST_RELEASE_CONFIRM="test.whoneedhelp.com:$candidate" \ +WNH_TEST_FORWARD_ONLY_CONFIRM="test.whoneedhelp.com:$candidate:forward-only" \ + ./scripts/test-release.sh apply whoneedhelp ``` +Omit `WNH_TEST_FORWARD_ONLY_CONFIRM` when the read-only plan reports +`migration_policy=application_safe`. The workflow refuses shared Caddy changes, +requires a fast-forward from the deployed test commit, preserves the single +test `.env`, and never addresses the production Compose project or database. +Do not use `deploy-up.sh` for an ordinary public test update on the small +server: that command intentionally includes `--build` and therefore builds the +release on the target host. + Test always uses its own `who_need_help_test` PostGIS container/volume. Local development can use Mailpit; a public test deployment must use its own SMTP password and a visibly test-specific sender identity. diff --git a/scripts/prepare-production-images.sh b/scripts/prepare-production-images.sh index 08e1f36..b7997be 100755 --- a/scripts/prepare-production-images.sh +++ b/scripts/prepare-production-images.sh @@ -17,8 +17,8 @@ for command in docker gzip jq sha256sum; do } done -if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then - echo "Refusing to build production images from a dirty tracked checkout." >&2 +if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "Refusing to build production images from a dirty checkout." >&2 exit 2 fi diff --git a/scripts/prepare-production-release.sh b/scripts/prepare-production-release.sh index eadc152..81973ea 100755 --- a/scripts/prepare-production-release.sh +++ b/scripts/prepare-production-release.sh @@ -4,8 +4,8 @@ umask 077 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then - echo "Refusing to package a release from a dirty tracked checkout." >&2 +if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "Refusing to package a release from a dirty checkout." >&2 exit 1 fi diff --git a/scripts/prepare-test-images.sh b/scripts/prepare-test-images.sh new file mode 100755 index 0000000..abb948d --- /dev/null +++ b/scripts/prepare-test-images.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +source_env=${1:-} + +if [[ -z "$source_env" || ! -f "$source_env" ]]; then + echo "Usage: $0 TEST_ENV_FILE" >&2 + exit 2 +fi + +for command in docker gzip jq realpath sha256sum; do + command -v "$command" >/dev/null 2>&1 || { + echo "Required command is unavailable: $command" >&2 + exit 2 + } +done + +if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "Refusing to build test images from a dirty checkout." >&2 + exit 2 +fi + +commit=$(git -C "$ROOT" rev-parse --verify HEAD) +short_commit=${commit:0:12} +artifact_root=${WNH_TEST_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/test-releases"} +case "$artifact_root" in + /*) ;; + *) + echo "WNH_TEST_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2 + exit 2 + ;; +esac +mkdir -p "$artifact_root" +artifact_root=$(realpath --canonicalize-existing "$artifact_root") +release_dir="$artifact_root/$commit" +archive="$release_dir/who_need_help-$commit-test-images-linux-amd64.tar.gz" +checksum="$archive.sha256" +manifest="$release_dir/who_need_help-$commit-test-images.manifest" + +mkdir -p "$release_dir" +chmod 700 "$artifact_root" "$release_dir" + +build_env=$(mktemp "$release_dir/.test-image-build.XXXXXX") +cleanup() { + trap - EXIT HUP INT TERM + rm -f "$build_env" +} +trap cleanup EXIT HUP INT TERM +install -m 600 "$source_env" "$build_env" + +read_value() { + local key=$1 + awk -v key="$key" ' + index($0, key "=") == 1 { + print substr($0, length(key) + 2) + found = 1 + exit + } + END { if (!found) exit 1 } + ' "$build_env" +} + +replace_value() { + local key=$1 value=$2 temporary + temporary=$(mktemp "$release_dir/.test-image-env.XXXXXX") + chmod 600 "$temporary" + awk -v key="$key" -v value="$value" ' + index($0, key "=") == 1 { print key "=" value; found = 1; next } + { print } + END { if (!found) exit 1 } + ' "$build_env" >"$temporary" + mv "$temporary" "$build_env" + chmod 600 "$build_env" +} + +[[ "$(read_value DEPLOYMENT_ENV)" == test ]] || { + echo "The image build input is not a test environment." >&2 + exit 2 +} +[[ "$(read_value DATABASE_MODE)" == container ]] || { + echo "The verified test image workflow expects DATABASE_MODE=container." >&2 + exit 2 +} + +replace_value APP_IMAGE "who-need-help:test-$short_commit" +replace_value SOCKET_PROXY_IMAGE \ + "who-need-help:socket-proxy-test-$short_commit" +replace_value POSTGIS_IMAGE "who-need-help:postgis-test-$short_commit" + +topology=$(read_value APP_TOPOLOGY) +case "$topology" in + compact) + build_services=(migrate db) + ;; + split) + build_services=(docker-api-proxy proxy migrate db) + ;; + *) + echo "APP_TOPOLOGY must be compact or split." >&2 + exit 2 + ;; +esac + +app_image=$(read_value APP_IMAGE) +postgis_image=$(read_value POSTGIS_IMAGE) +images=("$app_image" "$postgis_image") +if [[ "$topology" == split ]]; then + socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE) + proxy_image=$( + "$ROOT/scripts/compose.sh" "$build_env" config --format json | + jq -er '.services.proxy.image' + ) + images+=("$socket_proxy_image" "$proxy_image") +fi + +if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then + [[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || { + echo "The test image package is incomplete; refusing to overwrite it." >&2 + exit 2 + } + ( + cd "$release_dir" + sha256sum --check "$(basename -- "$checksum")" >/dev/null + ) + echo "Test image package already exists; verifying all metadata." +else + "$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}" + + manifest_tmp=$(mktemp "$release_dir/.test-images-manifest.XXXXXX") + archive_tmp=$(mktemp "$release_dir/.test-images-archive.XXXXXX") + trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM + + { + printf 'format=1\n' + printf 'deployment=test\n' + printf 'commit=%s\n' "$commit" + printf 'platform=linux/amd64\n' + printf 'topology=%s\n' "$topology" + printf 'image_count=%s\n' "${#images[@]}" + for image in "${images[@]}"; do + platform=$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image") + [[ "$platform" == linux/amd64 ]] || { + echo "Test image has an unexpected platform: $image ($platform)" >&2 + exit 2 + } + image_id=$(docker image inspect --format '{{.Id}}' "$image") + printf 'image=%s|%s\n' "$image" "$image_id" + done + } >"$manifest_tmp" + + docker save "${images[@]}" | gzip -n -9 >"$archive_tmp" + mv "$archive_tmp" "$archive" + mv "$manifest_tmp" "$manifest" + chmod 600 "$archive" "$manifest" + hash=$(sha256sum "$archive" | awk '{print $1}') + printf '%s %s\n' "$hash" "$(basename -- "$archive")" >"$checksum" + chmod 600 "$checksum" +fi + +( + cd "$release_dir" + sha256sum --check "$(basename -- "$checksum")" >/dev/null +) +gzip -t "$archive" + +manifest_value() { + local key=$1 + awk -F= -v key="$key" ' + $1 == key { count += 1; value = substr($0, length(key) + 2) } + END { + if (count != 1) exit 1 + print value + } + ' "$manifest" +} + +[[ "$(manifest_value format)" == 1 ]] || { + echo "Test image manifest format is unsupported." >&2 + exit 2 +} +[[ "$(manifest_value deployment)" == test ]] || { + echo "Test image manifest has the wrong deployment identity." >&2 + exit 2 +} +[[ "$(manifest_value commit)" == "$commit" ]] || { + echo "Test image manifest commit does not match the current commit." >&2 + exit 2 +} +[[ "$(manifest_value platform)" == linux/amd64 ]] || { + echo "Test image manifest platform is not linux/amd64." >&2 + exit 2 +} +[[ "$(manifest_value topology)" == "$topology" ]] || { + echo "Test image manifest topology does not match the environment." >&2 + exit 2 +} +[[ "$(manifest_value image_count)" == "${#images[@]}" ]] || { + echo "Test image manifest count does not match the required images." >&2 + exit 2 +} +test "$(grep -c '^image=' "$manifest")" = "${#images[@]}" +for image in "${images[@]}"; do + awk -F'|' -v image="$image" ' + $1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 } + END { if (!found) exit 1 } + ' "$manifest" +done + +archive_hash=$(sha256sum "$archive" | awk '{print $1}') +expected_checksum="$archive_hash $(basename -- "$archive")" +[[ "$(cat -- "$checksum")" == "$expected_checksum" ]] || { + echo "Test image checksum metadata does not name the exact archive." >&2 + exit 2 +} + +cleanup +printf 'Test image archive: %s\n' "$archive" +printf 'Image archive checksum: %s\n' "$checksum" +printf 'Image manifest: %s\n' "$manifest" diff --git a/scripts/production-release-drill.sh b/scripts/production-release-drill.sh index 07d6e79..6b8fc29 100755 --- a/scripts/production-release-drill.sh +++ b/scripts/production-release-drill.sh @@ -157,7 +157,7 @@ cat >"$mock_bin/git" <<'EOF' set -eu case " $* " in *' symbolic-ref --quiet --short HEAD '*) exit 1 ;; - *' status --porcelain --untracked-files=no '*) exit 0 ;; + *' status --porcelain --untracked-files=normal '*) exit 0 ;; *' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;; *' rev-parse refs/wnh/releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; *' bundle verify '*) exit 0 ;; diff --git a/scripts/production-release-remote.sh b/scripts/production-release-remote.sh index 9ab970f..53d4544 100755 --- a/scripts/production-release-remote.sh +++ b/scripts/production-release-remote.sh @@ -83,8 +83,8 @@ current_commit=$(git -C "$root" rev-parse --verify HEAD) echo "Production checkout must be on main or detached at the deployed commit." >&2 exit 2 } -[[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || { - echo "Production checkout has tracked modifications." >&2 +[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || { + echo "Production checkout has uncommitted files." >&2 exit 2 } diff --git a/scripts/production-release.sh b/scripts/production-release.sh index c57777e..e31e3f4 100755 --- a/scripts/production-release.sh +++ b/scripts/production-release.sh @@ -100,8 +100,8 @@ if [[ "$action" == "plan" ]]; then exit 0 fi -if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then - echo "Refusing to release a dirty tracked checkout." >&2 +if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "Refusing to release a dirty checkout." >&2 exit 2 fi diff --git a/scripts/quality.sh b/scripts/quality.sh index 87a3f11..aaffd37 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -144,6 +144,7 @@ echo "Checking release migration compatibility policy" echo "Checking isolated production release orchestration" ./scripts/production-release-drill.sh +./scripts/test-release-drill.sh echo "Checking Dockerfiles with Hadolint 2.14.0" for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \ @@ -1598,6 +1599,7 @@ docker run --rm \ "$python_runtime_image" python test/scripts/install_production_external_monitor_test.py python3 test/scripts/production_release_artifact_root_test.py python3 test/scripts/production_release_clean_test.py +python3 test/scripts/test_release_artifact_root_test.py docker run --rm \ --volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \ "$python_runtime_image" python -c \ diff --git a/scripts/set-deployment-revision.sh b/scripts/set-deployment-revision.sh index 3caa023..036907f 100755 --- a/scripts/set-deployment-revision.sh +++ b/scripts/set-deployment-revision.sh @@ -29,8 +29,8 @@ if [ "$(stat -c '%a' "$env_file")" != 600 ]; then exit 1 fi -if [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]; then - echo "Refusing to select a deployment revision from a dirty tracked checkout." >&2 +if [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]; then + echo "Refusing to select a deployment revision from a dirty checkout." >&2 exit 1 fi diff --git a/scripts/test-release-drill.sh b/scripts/test-release-drill.sh new file mode 100755 index 0000000..68b5f6e --- /dev/null +++ b/scripts/test-release-drill.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd" +mkdir -p "$ROOT/output" +run_dir=$(mktemp -d "$ROOT/output/test-release-drill.XXXXXX") +fixture="$run_dir/test" +mock_bin="$run_dir/mock-bin" +remote_root=/srv/who_need_help-test +current_commit=1111111111111111111111111111111111111111 +target_commit=2222222222222222222222222222222222222222 +release_confirmation="test.whoneedhelp.com:$target_commit" +forward_confirmation="test.whoneedhelp.com:$target_commit:forward-only" + +cleanup() { + status=$? + trap - EXIT HUP INT TERM + if [[ "$status" -ne 0 ]]; then + for output in "$run_dir"/*.out; do + [[ -f "$output" ]] || continue + printf '\n--- %s ---\n' "$(basename -- "$output")" >&2 + sed -n '1,240p' "$output" >&2 + done + fi + find "$run_dir" -xdev -depth -delete 2>/dev/null || true + exit "$status" +} +trap cleanup EXIT HUP INT TERM + +install -d -m 700 \ + "$fixture/.git" \ + "$fixture/scripts" \ + "$fixture/output/releases/incoming" \ + "$fixture/output/backups/test" \ + "$mock_bin" + +write_old_env() { + install -m 600 /dev/null "$fixture/.env" + printf '%s\n' \ + 'DEPLOYMENT_ENV=test' \ + 'COMPOSE_PROJECT_NAME=who_need_help_test' \ + 'DATABASE_MODE=container' \ + 'APP_TOPOLOGY=compact' \ + 'PHX_HOST=test.whoneedhelp.com' \ + 'WNH_BASE_URL=https://test.whoneedhelp.com' \ + "APP_IMAGE=who-need-help:test-${current_commit:0:12}" \ + "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-${current_commit:0:12}" \ + "POSTGIS_IMAGE=who-need-help:postgis-test-${current_commit:0:12}" \ + >"$fixture/.env" +} +write_old_env + +bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle" +printf 'isolated test release drill bundle\n' >"$bundle" +bundle_hash=$(sha256sum "$bundle" | awk '{print $1}') +printf '%s %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256" + +image_archive="$fixture/output/releases/incoming/who_need_help-$target_commit-test-images-linux-amd64.tar.gz" +printf 'isolated test release drill image archive\n' | gzip -n >"$image_archive" +image_hash=$(sha256sum "$image_archive" | awk '{print $1}') +printf '%s %s\n' "$image_hash" "$(basename -- "$image_archive")" \ + >"$image_archive.sha256" + +app_config="$run_dir/app-image-config.json" +db_config="$run_dir/db-image-config.json" +printf '%s\n' \ + '{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}' \ + >"$app_config" +printf '%s\n' \ + '{"architecture":"amd64","os":"linux","variant":"test-db","rootfs":{"type":"layers","diff_ids":[]}}' \ + >"$db_config" +app_image_id="sha256:$(sha256sum "$app_config" | awk '{print $1}')" +db_image_id="sha256:$(sha256sum "$db_config" | awk '{print $1}')" +[[ "$app_image_id" =~ ^sha256:[0-9a-f]{64}$ ]] +[[ "$db_image_id" =~ ^sha256:[0-9a-f]{64}$ ]] + +image_manifest="$fixture/output/releases/incoming/who_need_help-$target_commit-test-images.manifest" +printf '%s\n' \ + 'format=1' \ + 'deployment=test' \ + "commit=$target_commit" \ + 'platform=linux/amd64' \ + 'topology=compact' \ + 'image_count=2' \ + "image=who-need-help:test-${target_commit:0:12}|$app_image_id" \ + "image=who-need-help:postgis-test-${target_commit:0:12}|$db_image_id" \ + >"$image_manifest" + +backup="$fixture/output/backups/test/pre-release.dump" +printf 'isolated test release drill backup\n' >"$backup" +backup_hash=$(sha256sum "$backup" | awk '{print $1}') +printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256" +chmod 600 "$bundle" "$bundle.sha256" "$image_archive" \ + "$image_archive.sha256" "$image_manifest" "$backup" "$backup.sha256" + +for script in validate-test-env.sh verify-realtime-cluster.sh \ + verify-beam-runtime.sh check-database.sh; do + install -m 755 /dev/null "$fixture/scripts/$script" + printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/$script" +done + +install -m 755 /dev/null "$fixture/scripts/restore-drill-compose.sh" +printf '%s\n' \ + '#!/bin/sh' \ + 'set -eu' \ + "printf 'restore-drill:%s\\n' \"\$1\" >>\"\$MOCK_COMMAND_LOG\"" \ + >"$fixture/scripts/restore-drill-compose.sh" + +install -m 755 /dev/null "$fixture/scripts/set-deployment-revision.sh" +cat >"$fixture/scripts/set-deployment-revision.sh" <<'EOF' +#!/bin/sh +set -eu +env_file=$1 +short=${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}} +sed -i \ + -e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:test-$short|" \ + -e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-$short|" \ + -e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-test-$short|" \ + "$env_file" +EOF + +install -m 755 /dev/null "$fixture/scripts/compose.sh" +cat >"$fixture/scripts/compose.sh" <<'EOF' +#!/bin/sh +set -eu +env_file=$1 +shift +case "$*" in + 'config --quiet') exit 0 ;; + 'ps -q db') printf 'db-1\n'; exit 0 ;; + 'ps -q app') printf 'app-1\n'; exit 0 ;; + 'stop app') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + 'up -d --no-build --wait db') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + 'run --rm --no-deps --interactive=false migrate') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + exit 0 + ;; + 'up -d --no-deps --no-build --force-recreate --wait app') + printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" + if [ "${MOCK_FAIL_APP_UP:-}" = once ] && + [ ! -e "$MOCK_FAIL_APP_MARKER" ]; then + : >"$MOCK_FAIL_APP_MARKER" + exit 23 + fi + exit 0 + ;; +esac +printf 'Unexpected compose invocation: %s\n' "$*" >&2 +exit 1 +EOF + +printf '%s\n' "$current_commit" >"$fixture/git-state" +install -m 755 /dev/null "$mock_bin/git" +cat >"$mock_bin/git" <<'EOF' +#!/bin/sh +set -eu +case " $* " in + *' status --porcelain --untracked-files=normal '*) exit 0 ;; + *' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;; + *' rev-parse refs/wnh/test-releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; + *' bundle verify '*) exit 0 ;; + *' bundle list-heads '*) printf '%s HEAD\n' "$MOCK_TARGET_COMMIT"; exit 0 ;; + *' fetch '*) printf 'git:%s\n' "$*" >>"$MOCK_COMMAND_LOG"; exit 0 ;; + *' merge-base --is-ancestor '*) exit 0 ;; + *' show refs/wnh/test-releases/'*':scripts/release-migration-policy.sh '*) + cat <<'POLICY' +#!/bin/sh +set -eu +printf 'migration_policy=%s\n' "$MOCK_MIGRATION_POLICY" +printf 'migration_versions=20260101000000:%s\n' "$MOCK_MIGRATION_POLICY" +printf 'migration_count=1\n' +POLICY + exit 0 + ;; + *' checkout --detach refs/wnh/test-releases/'*) + printf '%s\n' "$MOCK_TARGET_COMMIT" >"$MOCK_GIT_STATE" + exit 0 + ;; + *" checkout --detach $MOCK_CURRENT_COMMIT "*) + printf '%s\n' "$MOCK_CURRENT_COMMIT" >"$MOCK_GIT_STATE" + exit 0 + ;; +esac +printf 'Unexpected git invocation: %s\n' "$*" >&2 +exit 1 +EOF + +install -m 755 /dev/null "$mock_bin/docker" +cat >"$mock_bin/docker" <<'EOF' +#!/bin/sh +set -eu +if [ "$1" = inspect ]; then + case "$3" in + '{{.State.Status}}') printf 'running\n' ;; + '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;; + '{{.Config.Image}}') + if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then + printf 'who-need-help:test-wrong\n' + elif [ "$4" = db-1 ]; then + awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0, index($0, "=") + 1)}' \ + "$MOCK_ENV_FILE" + else + awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \ + "$MOCK_ENV_FILE" + fi + ;; + '{{.Image}}') + if [ "$4" = db-1 ]; then + printf '%s\n' "$DB_IMAGE_ID" + else + printf '%s\n' "$APP_IMAGE_ID" + fi + ;; + *) exit 1 ;; + esac + exit 0 +fi +if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = --format ]; then + image=$5 + case "$4" in + '{{.Id}}') + case "$image" in + who-need-help:postgis-test-*) printf '%s\n' "$DB_IMAGE_ID" ;; + *) printf '%s\n' "$APP_IMAGE_ID" ;; + esac + ;; + '{{.Os}}/{{.Architecture}}') printf 'linux/amd64\n' ;; + *) exit 1 ;; + esac + exit 0 +fi +if [ "$1" = load ]; then + cat >/dev/null + printf 'docker:load\n' >>"$MOCK_COMMAND_LOG" + exit 0 +fi +printf 'Unexpected docker invocation: %s\n' "$*" >&2 +exit 1 +EOF + +for command in curl jq pg_restore; do + install -m 755 /dev/null "$mock_bin/$command" + printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" +done + +touch "$fixture/mock-commands.log" +chmod 600 "$fixture/mock-commands.log" "$fixture/git-state" + +container_env=( + --env "MOCK_TARGET_COMMIT=$target_commit" + --env "MOCK_CURRENT_COMMIT=$current_commit" + --env "MOCK_GIT_STATE=$remote_root/git-state" + --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" + --env "MOCK_ENV_FILE=$remote_root/.env" + --env "APP_IMAGE_ID=$app_image_id" + --env "DB_IMAGE_ID=$db_image_id" + --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +) +container_mounts=( + --volume "$fixture:$remote_root" + --volume "$mock_bin:/mock-bin:ro" + --volume "$ROOT/scripts/test-release-remote.sh:/runner/test-release-remote.sh:ro" +) + +run_release() { + local policy=$1 + shift + docker run --rm \ + --network none \ + --user "$(id -u):$(id -g)" \ + --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,noexec \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --env "MOCK_MIGRATION_POLICY=$policy" \ + --env "WNH_TEST_RELEASE_CONFIRM=$release_confirmation" \ + --env "WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation" \ + "$@" \ + "${container_env[@]}" \ + "${container_mounts[@]}" \ + "$BASE_IMAGE" \ + bash -c 'bash -s -- "$@" < /runner/test-release-remote.sh' _ \ + apply "$remote_root" test.whoneedhelp.com \ + "$remote_root/output/releases/incoming/$(basename -- "$bundle")" \ + "$target_commit" \ + "$remote_root/output/backups/test/$(basename -- "$backup")" \ + "$policy" \ + "$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \ + "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")" +} + +run_release forward_only >"$run_dir/forward-success.out" +grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null +grep -Fx "POSTGIS_IMAGE=who-need-help:postgis-test-${target_commit:0:12}" "$fixture/.env" >/dev/null +grep -Fx 'docker:load' "$fixture/mock-commands.log" >/dev/null +grep -F 'restore-drill:' "$fixture/mock-commands.log" >/dev/null +grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null +grep -F 'compose:run --rm --no-deps --interactive=false migrate' \ + "$fixture/mock-commands.log" >/dev/null +grep -F 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ + "$fixture/mock-commands.log" >/dev/null +if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then + echo "Test release drill unexpectedly built images on the target host." >&2 + exit 1 +fi +grep -R -F 'status=success' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null + +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +rm -f "$fixture/app-up-failed" +set +e +run_release forward_only \ + --env MOCK_FAIL_APP_UP=once \ + --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ + >"$run_dir/forward-failure.out" 2>&1 +forward_status=$? +set -e +[[ "$forward_status" -ne 0 ]] || { + echo "Forward-only test drill did not surface the startup failure." >&2 + exit 1 +} +grep -F 'automatic old-image restart is blocked' "$run_dir/forward-failure.out" >/dev/null +grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null +test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ + "$fixture/mock-commands.log")" = 1 + +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +rm -f "$fixture/app-up-failed" +set +e +run_release application_safe \ + --env MOCK_FAIL_APP_UP=once \ + --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ + >"$run_dir/safe-failure.out" 2>&1 +safe_status=$? +set -e +[[ "$safe_status" -ne 0 ]] || { + echo "Application-safe test drill did not surface the startup failure." >&2 + exit 1 +} +grep -F 'restoring the previous revision' "$run_dir/safe-failure.out" >/dev/null +grep -Fx "APP_IMAGE=who-need-help:test-${current_commit:0:12}" "$fixture/.env" >/dev/null +grep -Fx "$current_commit" "$fixture/git-state" >/dev/null +test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ + "$fixture/mock-commands.log")" = 2 + +echo "Isolated test release success/forward-only/safe-recovery drill passed." diff --git a/scripts/test-release-remote.sh b/scripts/test-release-remote.sh new file mode 100755 index 0000000..05fd188 --- /dev/null +++ b/scripts/test-release-remote.sh @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +action=${1:-} +root=${2:-/srv/who_need_help-test} +expected_domain=${3:-test.whoneedhelp.com} +bundle=${4:-} +target_commit=${5:-} +backup=${6:-} +expected_migration_policy=${7:-} +image_archive=${8:-} +image_manifest=${9:-} + +usage() { + echo "Usage: $0 plan /srv/who_need_help-test test.whoneedhelp.com" >&2 + echo " $0 apply /srv/who_need_help-test test.whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2 +} + +case "$action" in + plan | apply) ;; + *) usage; exit 2 ;; +esac + +for command in curl docker git gzip jq pg_restore realpath sha256sum; do + command -v "$command" >/dev/null 2>&1 || { + echo "Required test release command is unavailable: $command" >&2 + exit 2 + } +done + +root=$(realpath --canonicalize-existing "$root") +[[ "$root" == /srv/who_need_help-test ]] || { + echo "Refusing a test release outside /srv/who_need_help-test." >&2 + exit 2 +} + +env_file="$root/.env" +[[ -f "$env_file" && "$(stat -c '%a' "$env_file")" == 600 ]] || { + echo "Test .env is missing or does not have mode 0600." >&2 + exit 2 +} + +read_value() { + local key=$1 + awk -v key="$key" ' + index($0, key "=") == 1 { + print substr($0, length(key) + 2) + found = 1 + exit + } + END { if (!found) exit 1 } + ' "$env_file" +} + +deployment_environment=$(read_value DEPLOYMENT_ENV) +compose_project=$(read_value COMPOSE_PROJECT_NAME) +database_mode=$(read_value DATABASE_MODE) +app_topology=$(read_value APP_TOPOLOGY) +phx_host=$(read_value PHX_HOST) +public_origin=$(read_value WNH_BASE_URL) +current_commit=$(git -C "$root" rev-parse --verify HEAD) + +[[ "$deployment_environment" == test ]] || { + echo "DEPLOYMENT_ENV is not test." >&2 + exit 2 +} +[[ "$compose_project" == who_need_help_test ]] || { + echo "Unexpected test Compose project." >&2 + exit 2 +} +[[ "$database_mode" == container ]] || { + echo "The verified test workflow expects DATABASE_MODE=container." >&2 + exit 2 +} +[[ "$phx_host" == "$expected_domain" && + "$public_origin" == "https://$expected_domain" ]] || { + echo "Test origin does not match the expected domain." >&2 + exit 2 +} +[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || { + echo "Test checkout has uncommitted files." >&2 + exit 2 +} + +"$root/scripts/compose.sh" "$env_file" config --quiet + +case "$app_topology" in + compact) + expected_services=(app) + runtime_services=(app) + ;; + split) + expected_services=(web worker) + runtime_services=(docker-api-proxy proxy web worker) + ;; + *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;; +esac + +for service in db "${expected_services[@]}"; do + mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") + [[ ${#containers[@]} -gt 0 ]] || { + echo "Test service is not running: $service" >&2 + exit 2 + } + for container in "${containers[@]}"; do + state=$(docker inspect --format '{{.State.Status}}' "$container") + health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + [[ "$state" == running && "$health" == healthy ]] || { + echo "Test container is not healthy: $service" >&2 + exit 2 + } + done +done + +curl --fail --silent --show-error --max-time 15 \ + "https://$expected_domain/healthz/ready" >/dev/null + +printf 'Test checkout: %s\n' "$root" +printf 'Current commit: %s\n' "$current_commit" +printf 'Compose project: %s\n' "$compose_project" +printf 'Topology: %s\n' "$app_topology" +printf 'Database mode: %s\n' "$database_mode" +printf 'Public readiness: passed\n' +df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root" + +if [[ "$action" == plan ]]; then + echo "Read-only test release scope check passed." + exit 0 +fi + +if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" || + -z "$expected_migration_policy" || -z "$image_archive" || + -z "$image_manifest" ]]; then + usage + exit 2 +fi + +expected_confirmation="$expected_domain:$target_commit" +[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$expected_confirmation" ]] || { + echo "Set WNH_TEST_RELEASE_CONFIRM=$expected_confirmation for the approved test release." >&2 + exit 2 +} + +for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \ + "$image_archive" "$image_archive.sha256" "$image_manifest"; do + [[ -f "$required_file" ]] || { + echo "Required test release evidence is missing: $required_file" >&2 + exit 2 + } +done + +( + cd "$(dirname -- "$bundle")" + sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null +) +( + cd "$(dirname -- "$backup")" + sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null +) +pg_restore --list "$backup" >/dev/null +( + cd "$(dirname -- "$image_archive")" + sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null +) +gzip -t "$image_archive" +grep -Fx 'format=1' "$image_manifest" >/dev/null +grep -Fx 'deployment=test' "$image_manifest" >/dev/null +grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null +grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null +git -C "$root" bundle verify "$bundle" >/dev/null + +bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') +[[ "$bundle_head" == "$target_commit" ]] || { + echo "Bundle HEAD does not match the approved test commit." >&2 + exit 2 +} + +release_ref="refs/wnh/test-releases/$target_commit" +git -C "$root" fetch "$bundle" "HEAD:$release_ref" +[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || { + echo "Fetched test release ref does not match the approved commit." >&2 + exit 1 +} +git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { + echo "Test updates must be a fast-forward from the deployed commit." >&2 + exit 1 +} + +policy_script=$(mktemp) +trap 'rm -f "$policy_script"' EXIT HUP INT TERM +git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script" +chmod 700 "$policy_script" +migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root") +rm -f "$policy_script" +trap - EXIT HUP INT TERM +printf '%s\n' "$migration_policy_output" +migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output") +[[ "$migration_policy" == "$expected_migration_policy" ]] || { + echo "Remote migration policy does not match the locally approved policy." >&2 + exit 2 +} + +if [[ "$migration_policy" == forward_only ]]; then + forward_confirmation="$expected_domain:$target_commit:forward-only" + [[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || { + echo "Set WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation for this test schema boundary." >&2 + exit 2 + } +fi + +release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}" +release_dir="$root/output/releases/$release_id" +mkdir -p "$release_dir" +chmod 700 "$root/output" "$root/output/releases" "$release_dir" +rollback_manifest="$release_dir/rollback-manifest.txt" +{ + printf 'previous_commit=%s\n' "$current_commit" + printf 'target_commit=%s\n' "$target_commit" + printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)" + printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)" + printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)" + printf 'migration_policy=%s\n' "$migration_policy" + printf 'database_backup=%s\n' "$backup" + printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'status=started\n' +} >"$rollback_manifest" +chmod 600 "$rollback_manifest" + +revision_changed=false +migration_started=false + +restore_previous_revision() { + local temporary + temporary=$(mktemp "$root/.env.test-release-rollback.XXXXXX") + chmod 600 "$temporary" + APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ + SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ + POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \ + awk ' + BEGIN { + replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"] + replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"] + replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"] + } + { + separator = index($0, "=") + key = separator > 1 ? substr($0, 1, separator - 1) : "" + print (key in replacement) ? key "=" replacement[key] : $0 + } + ' "$env_file" >"$temporary" + mv "$temporary" "$env_file" + chmod 600 "$env_file" + git -C "$root" checkout --detach "$current_commit" >/dev/null +} + +rollback_runtime() { + local status=$? + trap - EXIT HUP INT TERM + if [[ "$status" -ne 0 && "$revision_changed" == true && + "$migration_policy" == forward_only && "$migration_started" == true ]]; then + { + printf 'status=forward-only-release-failed\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'automatic_application_rollback=blocked\n' + } >>"$rollback_manifest" + echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2 + elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then + echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2 + restore_previous_revision + "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db || true + "$root/scripts/compose.sh" "$env_file" \ + up -d --no-deps --no-build --force-recreate --wait \ + "${runtime_services[@]}" || true + { + printf 'status=runtime-rolled-back\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >>"$rollback_manifest" + fi + exit "$status" +} +trap rollback_runtime EXIT HUP INT TERM + +gzip -dc "$image_archive" | docker load >/dev/null +git -C "$root" checkout --detach "$release_ref" >/dev/null +"$root/scripts/set-deployment-revision.sh" "$env_file" +revision_changed=true +"$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain" + +expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)") +if [[ "$app_topology" == split ]]; then + expected_images+=( + "$(read_value SOCKET_PROXY_IMAGE)" + "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" + ) +fi +grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null +test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}" +for image in "${expected_images[@]}"; do + expected_id=$(awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' "$image_manifest") + [[ -n "$expected_id" ]] || { + echo "Image manifest is missing the expected image: $image" >&2 + exit 2 + } + [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || { + echo "Loaded test image ID does not match the manifest: $image" >&2 + exit 2 + } + [[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || { + echo "Loaded test image is not linux/amd64: $image" >&2 + exit 2 + } +done + +"$root/scripts/restore-drill-compose.sh" "$backup" + +if [[ "$migration_policy" == forward_only ]]; then + echo "Stopping the old test application before the forward-only migration boundary." + "$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}" +fi + +"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db + +verify_service_image() { + local service=$1 expected_image=$2 require_health=$3 + local expected_image_id container state health configured_image running_image_id + + expected_image_id=$(docker image inspect --format '{{.Id}}' "$expected_image") + mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") + [[ ${#containers[@]} -gt 0 ]] || { + echo "Candidate test service has no container: $service" >&2 + return 1 + } + + for container in "${containers[@]}"; do + state=$(docker inspect --format '{{.State.Status}}' "$container") + health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") + running_image_id=$(docker inspect --format '{{.Image}}' "$container") + + [[ "$state" == running ]] || { + echo "Candidate test container is not running: $service" >&2 + return 1 + } + if [[ "$require_health" == true && "$health" != healthy ]]; then + echo "Candidate test container is not healthy: $service" >&2 + return 1 + fi + [[ "$configured_image" == "$expected_image" && + "$running_image_id" == "$expected_image_id" ]] || { + echo "Candidate test service does not use its approved image: $service" >&2 + return 1 + } + done +} + +verify_service_image db "$(read_value POSTGIS_IMAGE)" true +migration_started=true +"$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate +"$root/scripts/check-database.sh" "$env_file" /dev/null + +{ + printf 'status=success\n' + printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +} >>"$rollback_manifest" +revision_changed=false +trap - EXIT HUP INT TERM + +printf 'Test release completed: %s\n' "$target_commit" +printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest" +printf 'Database backup: %s\n' "$backup" diff --git a/scripts/test-release.sh b/scripts/test-release.sh new file mode 100755 index 0000000..00b884d --- /dev/null +++ b/scripts/test-release.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +action=${1:-plan} +ssh_target=${2:-whoneedhelp} +remote_root=${WNH_TEST_REMOTE_ROOT:-/srv/who_need_help-test} +production_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production} +expected_domain=${WNH_TEST_DOMAIN:-test.whoneedhelp.com} + +case "$action" in + plan | prepare | apply) ;; + *) + echo "Usage: $0 [plan|prepare|apply] [SSH_TARGET]" >&2 + exit 2 + ;; +esac + +for command in git gzip mktemp pg_restore realpath scp sha256sum ssh; do + command -v "$command" >/dev/null 2>&1 || { + echo "Required command is unavailable: $command" >&2 + exit 2 + } +done + +local_commit=$(git -C "$ROOT" rev-parse --verify HEAD) +remote_commit=$( + ssh -o BatchMode=yes "$ssh_target" \ + "git -C '$remote_root' rev-parse --verify HEAD" +) + +printf 'Local candidate commit: %s\n' "$local_commit" +printf 'Current test commit: %s\n' "$remote_commit" + +git -C "$ROOT" cat-file -e "$remote_commit^{commit}" 2>/dev/null || { + echo "The test commit is not present in the local object database." >&2 + exit 2 +} +git -C "$ROOT" merge-base --is-ancestor "$remote_commit" "$local_commit" || { + echo "The local candidate is not a fast-forward from the test commit." >&2 + exit 2 +} +printf 'Pending commits: %s\n' \ + "$(git -C "$ROOT" rev-list --count "$remote_commit..$local_commit")" + +legacy_edge_paths=(compose.edge.yaml deploy/caddy/Caddyfile) +if ! git -C "$ROOT" diff --quiet \ + "$remote_commit" "$local_commit" -- "${legacy_edge_paths[@]}"; then + echo "The test application release contains shared edge routing changes." >&2 + echo "Move route changes through the independent server-edge workflow." >&2 + exit 2 +fi +echo "Shared edge routing files are unchanged; the test release will not manage Caddy." + +migration_policy_output=$( + "$ROOT/scripts/release-migration-policy.sh" "$remote_commit" "$local_commit" +) +printf '%s\n' "$migration_policy_output" +migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output") + +plan_failed=0 +if ! ssh -o BatchMode=yes "$ssh_target" \ + "cd '$remote_root' && ./scripts/validate-test-env.sh .env '$expected_domain'"; then + plan_failed=1 +fi +if ! ssh -o BatchMode=yes "$ssh_target" \ + "cd '$remote_root' && ./scripts/validate-deployment-isolation.sh '$remote_root' '$production_root'"; then + plan_failed=1 +fi +if ! ssh -o BatchMode=yes "$ssh_target" \ + "bash -s -- plan '$remote_root' '$expected_domain'" \ + <"$ROOT/scripts/test-release-remote.sh"; then + plan_failed=1 +fi +if ! ssh -o BatchMode=yes "$ssh_target" \ + "cd '$remote_root' && ./scripts/check-environment-readiness.sh .env --require-server-release"; then + plan_failed=1 +fi + +if [[ "$plan_failed" -ne 0 ]]; then + echo "Test release plan has blocking checks; no remote state was changed." >&2 + exit 1 +fi + +if [[ "$action" == plan ]]; then + echo "Test release plan passed; no remote state was changed." + exit 0 +fi + +[[ -z "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]] || { + echo "Refusing to release a dirty checkout." >&2 + exit 2 +} + +artifact_root=${WNH_TEST_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/test-releases"} +case "$artifact_root" in + /*) ;; + *) + echo "WNH_TEST_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2 + exit 2 + ;; +esac +mkdir -p "$artifact_root" +artifact_root=$(realpath --canonicalize-existing "$artifact_root") + +WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT="$artifact_root" \ + "$ROOT/scripts/prepare-production-release.sh" +release_dir="$artifact_root/$local_commit" +bundle="$release_dir/who_need_help-$local_commit.bundle" +image_archive="$release_dir/who_need_help-$local_commit-test-images-linux-amd64.tar.gz" +image_checksum="$image_archive.sha256" +image_manifest="$release_dir/who_need_help-$local_commit-test-images.manifest" + +test_env=$(mktemp) +cleanup_test_env() { + trap - EXIT HUP INT TERM + rm -f "$test_env" +} +trap cleanup_test_env EXIT HUP INT TERM +scp -p "$ssh_target:$remote_root/.env" "$test_env" +chmod 600 "$test_env" +WNH_TEST_RELEASE_ARTIFACT_ROOT="$artifact_root" \ + "$ROOT/scripts/prepare-test-images.sh" "$test_env" +cleanup_test_env + +if [[ "$action" == prepare ]]; then + echo "Test release artifacts are prepared and verified locally; no remote state was changed." + exit 0 +fi + +remote_main=$(git -C "$ROOT" ls-remote origin refs/heads/main | awk '{print $1}') +[[ "$remote_main" == "$local_commit" ]] || { + echo "origin/main does not contain the exact approved test candidate." >&2 + echo "Expected: $local_commit" >&2 + echo "Observed: ${remote_main:-missing}" >&2 + exit 2 +} + +confirmation="$expected_domain:$local_commit" +[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$confirmation" ]] || { + echo "Test release execution requires exact approval:" >&2 + echo "WNH_TEST_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2 + exit 2 +} +if [[ "$migration_policy" == forward_only ]]; then + forward_confirmation="$expected_domain:$local_commit:forward-only" + [[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || { + echo "This test release contains forward-only migrations." >&2 + echo "WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation \\" >&2 + echo " WNH_TEST_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2 + exit 2 + } +fi + +remote_release_dir="$remote_root/output/releases/incoming" +remote_bundle="$remote_release_dir/$(basename -- "$bundle")" +remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")" +remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")" +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +remote_backup="$remote_root/output/backups/test/pre-$timestamp-${local_commit:0:12}.dump" + +ssh -o BatchMode=yes "$ssh_target" "install -d -m 700 '$remote_release_dir'" +scp -p \ + "$bundle" "$bundle.sha256" \ + "$image_archive" "$image_checksum" "$image_manifest" \ + "$ssh_target:$remote_release_dir/" + +ssh -o BatchMode=yes "$ssh_target" \ + "cd '$remote_root' && ./scripts/backup-compose.sh '$remote_backup'" + +local_backup_dir="$ROOT/output/test-backups/$timestamp-${local_commit:0:12}" +mkdir -p "$local_backup_dir" +chmod 700 "$ROOT/output" "$ROOT/output/test-backups" "$local_backup_dir" +scp -p \ + "$ssh_target:$remote_backup" \ + "$ssh_target:$remote_backup.sha256" \ + "$local_backup_dir/" +( + cd "$local_backup_dir" + sha256sum --check "$(basename -- "$remote_backup.sha256")" >/dev/null +) +pg_restore --list "$local_backup_dir/$(basename -- "$remote_backup")" >/dev/null +echo "Copied and independently verified the pre-release test backup outside the server." + +quoted_confirmation=$(printf '%q' "$confirmation") +quoted_forward_confirmation=$(printf '%q' "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}") +ssh -o BatchMode=yes "$ssh_target" \ + "WNH_TEST_RELEASE_CONFIRM=$quoted_confirmation WNH_TEST_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \ + <"$ROOT/scripts/test-release-remote.sh" + +echo "Test release and public health verification completed." diff --git a/test/scripts/production_release_artifact_root_test.py b/test/scripts/production_release_artifact_root_test.py index c12f2db..eab7b3d 100644 --- a/test/scripts/production_release_artifact_root_test.py +++ b/test/scripts/production_release_artifact_root_test.py @@ -91,6 +91,21 @@ class ProductionReleaseArtifactRootTest(unittest.TestCase): self.assertEqual(result.returncode, 2) self.assertIn("must be an absolute path", result.stderr) + def test_untracked_build_input_is_rejected(self): + (self.project / "untracked-build-input.txt").write_text( + "must not enter an immutable release\n", encoding="utf-8" + ) + + result = self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=self.artifact_env(), + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("dirty checkout", result.stderr) + def test_bundle_manifest_for_another_commit_is_rejected(self): self.run_command( [str(self.scripts / "prepare-production-release.sh")], diff --git a/test/scripts/test_release_artifact_root_test.py b/test/scripts/test_release_artifact_root_test.py new file mode 100644 index 0000000..5d0f43e --- /dev/null +++ b/test/scripts/test_release_artifact_root_test.py @@ -0,0 +1,178 @@ +import gzip +import hashlib +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +class TestReleaseArtifactRootTest(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.base = Path(self.tempdir.name) + self.project = self.base / "project" + self.scripts = self.project / "scripts" + self.scripts.mkdir(parents=True) + script = self.scripts / "prepare-test-images.sh" + shutil.copy2(ROOT / "scripts" / script.name, script) + script.chmod(0o755) + + self.run_command(["git", "init", "--quiet"], cwd=self.project) + self.run_command( + ["git", "config", "user.email", "test-release@example.invalid"], + cwd=self.project, + ) + self.run_command( + ["git", "config", "user.name", "Test release"], cwd=self.project + ) + self.run_command(["git", "add", "scripts"], cwd=self.project) + self.run_command( + ["git", "commit", "--quiet", "-m", "test fixture"], cwd=self.project + ) + self.commit = self.run_command( + ["git", "rev-parse", "HEAD"], cwd=self.project + ).stdout.strip() + self.artifact_root = self.base / "test-artifacts" + + self.fake_bin = self.base / "bin" + self.fake_bin.mkdir() + for name in ("docker", "jq"): + command = self.fake_bin / name + command.write_text("#!/bin/sh\nexit 97\n", encoding="utf-8") + command.chmod(0o755) + + self.test_env = self.base / "test.env" + self.test_env.write_text( + textwrap.dedent( + """\ + DEPLOYMENT_ENV=test + DATABASE_MODE=container + APP_TOPOLOGY=compact + APP_IMAGE=replace-me + SOCKET_PROXY_IMAGE=replace-me + POSTGIS_IMAGE=replace-me + """ + ), + encoding="utf-8", + ) + self.test_env.chmod(0o600) + + def tearDown(self): + self.tempdir.cleanup() + + def run_command(self, command, *, cwd=None, env=None, check=True): + return subprocess.run( + command, + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=check, + ) + + def environment(self): + env = os.environ.copy() + env["WNH_TEST_RELEASE_ARTIFACT_ROOT"] = str(self.artifact_root) + env["PATH"] = f"{self.fake_bin}:{env['PATH']}" + return env + + def write_existing_artifact(self): + release_dir = self.artifact_root / self.commit + release_dir.mkdir(parents=True) + archive = ( + release_dir + / f"who_need_help-{self.commit}-test-images-linux-amd64.tar.gz" + ) + with archive.open("wb") as output: + with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed: + compressed.write(b"verified test image archive fixture") + archive.chmod(0o600) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + checksum = Path(f"{archive}.sha256") + checksum.write_text(f"{digest} {archive.name}\n", encoding="utf-8") + checksum.chmod(0o600) + + short_commit = self.commit[:12] + manifest = release_dir / f"who_need_help-{self.commit}-test-images.manifest" + manifest.write_text( + textwrap.dedent( + f"""\ + format=1 + deployment=test + commit={self.commit} + platform=linux/amd64 + topology=compact + image_count=2 + image=who-need-help:test-{short_commit}|sha256:{'a' * 64} + image=who-need-help:postgis-test-{short_commit}|sha256:{'b' * 64} + """ + ), + encoding="utf-8", + ) + manifest.chmod(0o600) + return archive, manifest + + def test_existing_test_archive_is_verified_without_building(self): + archive, manifest = self.write_existing_artifact() + + result = self.run_command( + [str(self.scripts / "prepare-test-images.sh"), str(self.test_env)], + cwd=self.project, + env=self.environment(), + ) + + self.assertIn("verifying all metadata", result.stdout) + self.assertIn(str(archive), result.stdout) + + manifest.write_text( + manifest.read_text(encoding="utf-8").replace( + "deployment=test", "deployment=production" + ), + encoding="utf-8", + ) + rejected = self.run_command( + [str(self.scripts / "prepare-test-images.sh"), str(self.test_env)], + cwd=self.project, + env=self.environment(), + check=False, + ) + self.assertEqual(rejected.returncode, 2) + self.assertIn("wrong deployment identity", rejected.stderr) + + def test_relative_artifact_root_is_rejected(self): + env = self.environment() + env["WNH_TEST_RELEASE_ARTIFACT_ROOT"] = "relative/test-releases" + result = self.run_command( + [str(self.scripts / "prepare-test-images.sh"), str(self.test_env)], + cwd=self.project, + env=env, + check=False, + ) + + self.assertEqual(result.returncode, 2) + self.assertIn("must be an absolute path", result.stderr) + + def test_untracked_build_input_is_rejected(self): + (self.project / "untracked-build-input.txt").write_text( + "must not enter an immutable release\n", encoding="utf-8" + ) + + result = self.run_command( + [str(self.scripts / "prepare-test-images.sh"), str(self.test_env)], + cwd=self.project, + env=self.environment(), + check=False, + ) + + self.assertEqual(result.returncode, 2) + self.assertIn("dirty checkout", result.stderr) + + +if __name__ == "__main__": + unittest.main()