simpletest/who_need_help
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add isolated public test release workflow

Browse Source
This commit is contained in:
SimpleTest 2026-08-27 23:49:02 +03:00
parent 86260cead1
commit 4d09caf130
14 changed files with 1388 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/operations.md
View File

@ -361,14 +361,32 @@ TEST_GOOGLE_OAUTH_AUTHORIZED_PARTY_IDS=YOUR_TEST_ANDROID_CLIENT_ID \
./scripts/deploy-up.sh .env
```
For later test updates, check out the desired clean revision and update only
the image tags. Existing deployment secrets remain unchanged:
For later public test updates, release the exact clean revision with immutable
images built on the operator workstation. The test release workflow performs
a read-only scope plan, builds and verifies `linux/amd64` images outside the
4-GiB server, creates and copies an independently verified database backup,
runs a restore-and-migrate drill, and starts Compose with `--no-build`:
```bash
./scripts/set-deployment-revision.sh .env
./scripts/deploy-up.sh .env
./scripts/test-release.sh plan whoneedhelp
./scripts/test-release.sh prepare whoneedhelp
# Make the selected commit available as origin/main before apply. The apply
# gate verifies that origin/main equals the exact local SHA.
candidate=$(git rev-parse HEAD)
WNH_TEST_RELEASE_CONFIRM="test.whoneedhelp.com:$candidate" \
WNH_TEST_FORWARD_ONLY_CONFIRM="test.whoneedhelp.com:$candidate:forward-only" \
./scripts/test-release.sh apply whoneedhelp
```
Omit `WNH_TEST_FORWARD_ONLY_CONFIRM` when the read-only plan reports
`migration_policy=application_safe`. The workflow refuses shared Caddy changes,
requires a fast-forward from the deployed test commit, preserves the single
test `.env`, and never addresses the production Compose project or database.
Do not use `deploy-up.sh` for an ordinary public test update on the small
server: that command intentionally includes `--build` and therefore builds the
release on the target host.
Test always uses its own `who_need_help_test` PostGIS container/volume. Local
development can use Mailpit; a public test deployment must use its own SMTP
password and a visibly test-specific sender identity.

4
scripts/prepare-production-images.sh
View File

@ -17,8 +17,8 @@ for command in docker gzip jq sha256sum; do
}
done
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
echo "Refusing to build production images from a dirty tracked checkout." >&2
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
echo "Refusing to build production images from a dirty checkout." >&2
exit 2
fi

4
scripts/prepare-production-release.sh
View File

@ -4,8 +4,8 @@ umask 077
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
echo "Refusing to package a release from a dirty tracked checkout." >&2
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
echo "Refusing to package a release from a dirty checkout." >&2
exit 1
fi

221
scripts/prepare-test-images.sh Executable file
View File

@ -0,0 +1,221 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
source_env=${1:-}
if [[ -z "$source_env" || ! -f "$source_env" ]]; then
echo "Usage: $0 TEST_ENV_FILE" >&2
exit 2
fi
for command in docker gzip jq realpath sha256sum; do
command -v "$command" >/dev/null 2>&1 || {
echo "Required command is unavailable: $command" >&2
exit 2
}
done
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
echo "Refusing to build test images from a dirty checkout." >&2
exit 2
fi
commit=$(git -C "$ROOT" rev-parse --verify HEAD)
short_commit=${commit:0:12}
artifact_root=${WNH_TEST_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/test-releases"}
case "$artifact_root" in
/*) ;;
*)
echo "WNH_TEST_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
exit 2
;;
esac
mkdir -p "$artifact_root"
artifact_root=$(realpath --canonicalize-existing "$artifact_root")
release_dir="$artifact_root/$commit"
archive="$release_dir/who_need_help-$commit-test-images-linux-amd64.tar.gz"
checksum="$archive.sha256"
manifest="$release_dir/who_need_help-$commit-test-images.manifest"
mkdir -p "$release_dir"
chmod 700 "$artifact_root" "$release_dir"
build_env=$(mktemp "$release_dir/.test-image-build.XXXXXX")
cleanup() {
trap - EXIT HUP INT TERM
rm -f "$build_env"
}
trap cleanup EXIT HUP INT TERM
install -m 600 "$source_env" "$build_env"
read_value() {
local key=$1
awk -v key="$key" '
index($0, key "=") == 1 {
print substr($0, length(key) + 2)
found = 1
exit
}
END { if (!found) exit 1 }
' "$build_env"
}
replace_value() {
local key=$1 value=$2 temporary
temporary=$(mktemp "$release_dir/.test-image-env.XXXXXX")
chmod 600 "$temporary"
awk -v key="$key" -v value="$value" '
index($0, key "=") == 1 { print key "=" value; found = 1; next }
{ print }
END { if (!found) exit 1 }
' "$build_env" >"$temporary"
mv "$temporary" "$build_env"
chmod 600 "$build_env"
}
[[ "$(read_value DEPLOYMENT_ENV)" == test ]] || {
echo "The image build input is not a test environment." >&2
exit 2
}
[[ "$(read_value DATABASE_MODE)" == container ]] || {
echo "The verified test image workflow expects DATABASE_MODE=container." >&2
exit 2
}
replace_value APP_IMAGE "who-need-help:test-$short_commit"
replace_value SOCKET_PROXY_IMAGE \
"who-need-help:socket-proxy-test-$short_commit"
replace_value POSTGIS_IMAGE "who-need-help:postgis-test-$short_commit"
topology=$(read_value APP_TOPOLOGY)
case "$topology" in
compact)
build_services=(migrate db)
;;
split)
build_services=(docker-api-proxy proxy migrate db)
;;
*)
echo "APP_TOPOLOGY must be compact or split." >&2
exit 2
;;
esac
app_image=$(read_value APP_IMAGE)
postgis_image=$(read_value POSTGIS_IMAGE)
images=("$app_image" "$postgis_image")
if [[ "$topology" == split ]]; then
socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE)
proxy_image=$(
"$ROOT/scripts/compose.sh" "$build_env" config --format json |
jq -er '.services.proxy.image'
)
images+=("$socket_proxy_image" "$proxy_image")
fi
if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then
[[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || {
echo "The test image package is incomplete; refusing to overwrite it." >&2
exit 2
}
(
cd "$release_dir"
sha256sum --check "$(basename -- "$checksum")" >/dev/null
)
echo "Test image package already exists; verifying all metadata."
else
"$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}"
manifest_tmp=$(mktemp "$release_dir/.test-images-manifest.XXXXXX")
archive_tmp=$(mktemp "$release_dir/.test-images-archive.XXXXXX")
trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM
{
printf 'format=1\n'
printf 'deployment=test\n'
printf 'commit=%s\n' "$commit"
printf 'platform=linux/amd64\n'
printf 'topology=%s\n' "$topology"
printf 'image_count=%s\n' "${#images[@]}"
for image in "${images[@]}"; do
platform=$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")
[[ "$platform" == linux/amd64 ]] || {
echo "Test image has an unexpected platform: $image ($platform)" >&2
exit 2
}
image_id=$(docker image inspect --format '{{.Id}}' "$image")
printf 'image=%s|%s\n' "$image" "$image_id"
done
} >"$manifest_tmp"
docker save "${images[@]}" | gzip -n -9 >"$archive_tmp"
mv "$archive_tmp" "$archive"
mv "$manifest_tmp" "$manifest"
chmod 600 "$archive" "$manifest"
hash=$(sha256sum "$archive" | awk '{print $1}')
printf '%s %s\n' "$hash" "$(basename -- "$archive")" >"$checksum"
chmod 600 "$checksum"
fi
(
cd "$release_dir"
sha256sum --check "$(basename -- "$checksum")" >/dev/null
)
gzip -t "$archive"
manifest_value() {
local key=$1
awk -F= -v key="$key" '
$1 == key { count += 1; value = substr($0, length(key) + 2) }
END {
if (count != 1) exit 1
print value
}
' "$manifest"
}
[[ "$(manifest_value format)" == 1 ]] || {
echo "Test image manifest format is unsupported." >&2
exit 2
}
[[ "$(manifest_value deployment)" == test ]] || {
echo "Test image manifest has the wrong deployment identity." >&2
exit 2
}
[[ "$(manifest_value commit)" == "$commit" ]] || {
echo "Test image manifest commit does not match the current commit." >&2
exit 2
}
[[ "$(manifest_value platform)" == linux/amd64 ]] || {
echo "Test image manifest platform is not linux/amd64." >&2
exit 2
}
[[ "$(manifest_value topology)" == "$topology" ]] || {
echo "Test image manifest topology does not match the environment." >&2
exit 2
}
[[ "$(manifest_value image_count)" == "${#images[@]}" ]] || {
echo "Test image manifest count does not match the required images." >&2
exit 2
}
test "$(grep -c '^image=' "$manifest")" = "${#images[@]}"
for image in "${images[@]}"; do
awk -F'|' -v image="$image" '
$1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 }
END { if (!found) exit 1 }
' "$manifest"
done
archive_hash=$(sha256sum "$archive" | awk '{print $1}')
expected_checksum="$archive_hash $(basename -- "$archive")"
[[ "$(cat -- "$checksum")" == "$expected_checksum" ]] || {
echo "Test image checksum metadata does not name the exact archive." >&2
exit 2
}
cleanup
printf 'Test image archive: %s\n' "$archive"
printf 'Image archive checksum: %s\n' "$checksum"
printf 'Image manifest: %s\n' "$manifest"

2
scripts/production-release-drill.sh
View File

@ -157,7 +157,7 @@ cat >"$mock_bin/git" <<'EOF'
set -eu
case " $* " in
*' symbolic-ref --quiet --short HEAD '*) exit 1 ;;
*' status --porcelain --untracked-files=no '*) exit 0 ;;
*' status --porcelain --untracked-files=normal '*) exit 0 ;;
*' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;;
*' rev-parse refs/wnh/releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
*' bundle verify '*) exit 0 ;;

4
scripts/production-release-remote.sh
View File

@ -83,8 +83,8 @@ current_commit=$(git -C "$root" rev-parse --verify HEAD)
echo "Production checkout must be on main or detached at the deployed commit." >&2
exit 2
}
[[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || {
echo "Production checkout has tracked modifications." >&2
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
echo "Production checkout has uncommitted files." >&2
exit 2
}

4
scripts/production-release.sh
View File

@ -100,8 +100,8 @@ if [[ "$action" == "plan" ]]; then
exit 0
fi
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
echo "Refusing to release a dirty tracked checkout." >&2
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
echo "Refusing to release a dirty checkout." >&2
exit 2
fi

2
scripts/quality.sh
View File

@ -144,6 +144,7 @@ echo "Checking release migration compatibility policy"
echo "Checking isolated production release orchestration"
./scripts/production-release-drill.sh
./scripts/test-release-drill.sh
echo "Checking Dockerfiles with Hadolint 2.14.0"
for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \
@ -1598,6 +1599,7 @@ docker run --rm \
"$python_runtime_image" python test/scripts/install_production_external_monitor_test.py
python3 test/scripts/production_release_artifact_root_test.py
python3 test/scripts/production_release_clean_test.py
python3 test/scripts/test_release_artifact_root_test.py
docker run --rm \
--volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \
"$python_runtime_image" python -c \

4
scripts/set-deployment-revision.sh
View File

@ -29,8 +29,8 @@ if [ "$(stat -c '%a' "$env_file")" != 600 ]; then
exit 1
fi
if [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]; then
echo "Refusing to select a deployment revision from a dirty tracked checkout." >&2
if [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]; then
echo "Refusing to select a deployment revision from a dirty checkout." >&2
exit 1
fi

358
scripts/test-release-drill.sh Executable file
View File

@ -0,0 +1,358 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd"
mkdir -p "$ROOT/output"
run_dir=$(mktemp -d "$ROOT/output/test-release-drill.XXXXXX")
fixture="$run_dir/test"
mock_bin="$run_dir/mock-bin"
remote_root=/srv/who_need_help-test
current_commit=1111111111111111111111111111111111111111
target_commit=2222222222222222222222222222222222222222
release_confirmation="test.whoneedhelp.com:$target_commit"
forward_confirmation="test.whoneedhelp.com:$target_commit:forward-only"
cleanup() {
status=$?
trap - EXIT HUP INT TERM
if [[ "$status" -ne 0 ]]; then
for output in "$run_dir"/*.out; do
[[ -f "$output" ]] || continue
printf '\n--- %s ---\n' "$(basename -- "$output")" >&2
sed -n '1,240p' "$output" >&2
done
fi
find "$run_dir" -xdev -depth -delete 2>/dev/null || true
exit "$status"
}
trap cleanup EXIT HUP INT TERM
install -d -m 700 \
"$fixture/.git" \
"$fixture/scripts" \
"$fixture/output/releases/incoming" \
"$fixture/output/backups/test" \
"$mock_bin"
write_old_env() {
install -m 600 /dev/null "$fixture/.env"
printf '%s\n' \
'DEPLOYMENT_ENV=test' \
'COMPOSE_PROJECT_NAME=who_need_help_test' \
'DATABASE_MODE=container' \
'APP_TOPOLOGY=compact' \
'PHX_HOST=test.whoneedhelp.com' \
'WNH_BASE_URL=https://test.whoneedhelp.com' \
"APP_IMAGE=who-need-help:test-${current_commit:0:12}" \
"SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-${current_commit:0:12}" \
"POSTGIS_IMAGE=who-need-help:postgis-test-${current_commit:0:12}" \
>"$fixture/.env"
}
write_old_env
bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle"
printf 'isolated test release drill bundle\n' >"$bundle"
bundle_hash=$(sha256sum "$bundle" | awk '{print $1}')
printf '%s %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256"
image_archive="$fixture/output/releases/incoming/who_need_help-$target_commit-test-images-linux-amd64.tar.gz"
printf 'isolated test release drill image archive\n' | gzip -n >"$image_archive"
image_hash=$(sha256sum "$image_archive" | awk '{print $1}')
printf '%s %s\n' "$image_hash" "$(basename -- "$image_archive")" \
>"$image_archive.sha256"
app_config="$run_dir/app-image-config.json"
db_config="$run_dir/db-image-config.json"
printf '%s\n' \
'{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}' \
>"$app_config"
printf '%s\n' \
'{"architecture":"amd64","os":"linux","variant":"test-db","rootfs":{"type":"layers","diff_ids":[]}}' \
>"$db_config"
app_image_id="sha256:$(sha256sum "$app_config" | awk '{print $1}')"
db_image_id="sha256:$(sha256sum "$db_config" | awk '{print $1}')"
[[ "$app_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]
[[ "$db_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]
image_manifest="$fixture/output/releases/incoming/who_need_help-$target_commit-test-images.manifest"
printf '%s\n' \
'format=1' \
'deployment=test' \
"commit=$target_commit" \
'platform=linux/amd64' \
'topology=compact' \
'image_count=2' \
"image=who-need-help:test-${target_commit:0:12}|$app_image_id" \
"image=who-need-help:postgis-test-${target_commit:0:12}|$db_image_id" \
>"$image_manifest"
backup="$fixture/output/backups/test/pre-release.dump"
printf 'isolated test release drill backup\n' >"$backup"
backup_hash=$(sha256sum "$backup" | awk '{print $1}')
printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256"
chmod 600 "$bundle" "$bundle.sha256" "$image_archive" \
"$image_archive.sha256" "$image_manifest" "$backup" "$backup.sha256"
for script in validate-test-env.sh verify-realtime-cluster.sh \
verify-beam-runtime.sh check-database.sh; do
install -m 755 /dev/null "$fixture/scripts/$script"
printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/$script"
done
install -m 755 /dev/null "$fixture/scripts/restore-drill-compose.sh"
printf '%s\n' \
'#!/bin/sh' \
'set -eu' \
"printf 'restore-drill:%s\\n' \"\$1\" >>\"\$MOCK_COMMAND_LOG\"" \
>"$fixture/scripts/restore-drill-compose.sh"
install -m 755 /dev/null "$fixture/scripts/set-deployment-revision.sh"
cat >"$fixture/scripts/set-deployment-revision.sh" <<'EOF'
#!/bin/sh
set -eu
env_file=$1
short=${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}
sed -i \
-e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:test-$short|" \
-e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-$short|" \
-e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-test-$short|" \
"$env_file"
EOF
install -m 755 /dev/null "$fixture/scripts/compose.sh"
cat >"$fixture/scripts/compose.sh" <<'EOF'
#!/bin/sh
set -eu
env_file=$1
shift
case "$*" in
'config --quiet') exit 0 ;;
'ps -q db') printf 'db-1\n'; exit 0 ;;
'ps -q app') printf 'app-1\n'; exit 0 ;;
'stop app')
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
exit 0
;;
'up -d --no-build --wait db')
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
exit 0
;;
'run --rm --no-deps --interactive=false migrate')
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
exit 0
;;
'up -d --no-deps --no-build --force-recreate --wait app')
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
if [ "${MOCK_FAIL_APP_UP:-}" = once ] &&
[ ! -e "$MOCK_FAIL_APP_MARKER" ]; then
: >"$MOCK_FAIL_APP_MARKER"
exit 23
fi
exit 0
;;
esac
printf 'Unexpected compose invocation: %s\n' "$*" >&2
exit 1
EOF
printf '%s\n' "$current_commit" >"$fixture/git-state"
install -m 755 /dev/null "$mock_bin/git"
cat >"$mock_bin/git" <<'EOF'
#!/bin/sh
set -eu
case " $* " in
*' status --porcelain --untracked-files=normal '*) exit 0 ;;
*' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;;
*' rev-parse refs/wnh/test-releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
*' bundle verify '*) exit 0 ;;
*' bundle list-heads '*) printf '%s HEAD\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
*' fetch '*) printf 'git:%s\n' "$*" >>"$MOCK_COMMAND_LOG"; exit 0 ;;
*' merge-base --is-ancestor '*) exit 0 ;;
*' show refs/wnh/test-releases/'*':scripts/release-migration-policy.sh '*)
cat <<'POLICY'
#!/bin/sh
set -eu
printf 'migration_policy=%s\n' "$MOCK_MIGRATION_POLICY"
printf 'migration_versions=20260101000000:%s\n' "$MOCK_MIGRATION_POLICY"
printf 'migration_count=1\n'
POLICY
exit 0
;;
*' checkout --detach refs/wnh/test-releases/'*)
printf '%s\n' "$MOCK_TARGET_COMMIT" >"$MOCK_GIT_STATE"
exit 0
;;
*" checkout --detach $MOCK_CURRENT_COMMIT "*)
printf '%s\n' "$MOCK_CURRENT_COMMIT" >"$MOCK_GIT_STATE"
exit 0
;;
esac
printf 'Unexpected git invocation: %s\n' "$*" >&2
exit 1
EOF
install -m 755 /dev/null "$mock_bin/docker"
cat >"$mock_bin/docker" <<'EOF'
#!/bin/sh
set -eu
if [ "$1" = inspect ]; then
case "$3" in
'{{.State.Status}}') printf 'running\n' ;;
'{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;;
'{{.Config.Image}}')
if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then
printf 'who-need-help:test-wrong\n'
elif [ "$4" = db-1 ]; then
awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0, index($0, "=") + 1)}' \
"$MOCK_ENV_FILE"
else
awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \
"$MOCK_ENV_FILE"
fi
;;
'{{.Image}}')
if [ "$4" = db-1 ]; then
printf '%s\n' "$DB_IMAGE_ID"
else
printf '%s\n' "$APP_IMAGE_ID"
fi
;;
*) exit 1 ;;
esac
exit 0
fi
if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = --format ]; then
image=$5
case "$4" in
'{{.Id}}')
case "$image" in
who-need-help:postgis-test-*) printf '%s\n' "$DB_IMAGE_ID" ;;
*) printf '%s\n' "$APP_IMAGE_ID" ;;
esac
;;
'{{.Os}}/{{.Architecture}}') printf 'linux/amd64\n' ;;
*) exit 1 ;;
esac
exit 0
fi
if [ "$1" = load ]; then
cat >/dev/null
printf 'docker:load\n' >>"$MOCK_COMMAND_LOG"
exit 0
fi
printf 'Unexpected docker invocation: %s\n' "$*" >&2
exit 1
EOF
for command in curl jq pg_restore; do
install -m 755 /dev/null "$mock_bin/$command"
printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command"
done
touch "$fixture/mock-commands.log"
chmod 600 "$fixture/mock-commands.log" "$fixture/git-state"
container_env=(
--env "MOCK_TARGET_COMMIT=$target_commit"
--env "MOCK_CURRENT_COMMIT=$current_commit"
--env "MOCK_GIT_STATE=$remote_root/git-state"
--env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log"
--env "MOCK_ENV_FILE=$remote_root/.env"
--env "APP_IMAGE_ID=$app_image_id"
--env "DB_IMAGE_ID=$db_image_id"
--env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
)
container_mounts=(
--volume "$fixture:$remote_root"
--volume "$mock_bin:/mock-bin:ro"
--volume "$ROOT/scripts/test-release-remote.sh:/runner/test-release-remote.sh:ro"
)
run_release() {
local policy=$1
shift
docker run --rm \
--network none \
--user "$(id -u):$(id -g)" \
--read-only \
--tmpfs /tmp:rw,nosuid,nodev,noexec \
--cap-drop ALL \
--security-opt no-new-privileges \
--env "MOCK_MIGRATION_POLICY=$policy" \
--env "WNH_TEST_RELEASE_CONFIRM=$release_confirmation" \
--env "WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation" \
"$@" \
"${container_env[@]}" \
"${container_mounts[@]}" \
"$BASE_IMAGE" \
bash -c 'bash -s -- "$@" < /runner/test-release-remote.sh' _ \
apply "$remote_root" test.whoneedhelp.com \
"$remote_root/output/releases/incoming/$(basename -- "$bundle")" \
"$target_commit" \
"$remote_root/output/backups/test/$(basename -- "$backup")" \
"$policy" \
"$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \
"$remote_root/output/releases/incoming/$(basename -- "$image_manifest")"
}
run_release forward_only >"$run_dir/forward-success.out"
grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null
grep -Fx "POSTGIS_IMAGE=who-need-help:postgis-test-${target_commit:0:12}" "$fixture/.env" >/dev/null
grep -Fx 'docker:load' "$fixture/mock-commands.log" >/dev/null
grep -F 'restore-drill:' "$fixture/mock-commands.log" >/dev/null
grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null
grep -F 'compose:run --rm --no-deps --interactive=false migrate' \
"$fixture/mock-commands.log" >/dev/null
grep -F 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
"$fixture/mock-commands.log" >/dev/null
if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then
echo "Test release drill unexpectedly built images on the target host." >&2
exit 1
fi
grep -R -F 'status=success' \
"$fixture/output/releases" --include rollback-manifest.txt >/dev/null
write_old_env
printf '%s\n' "$current_commit" >"$fixture/git-state"
: >"$fixture/mock-commands.log"
rm -f "$fixture/app-up-failed"
set +e
run_release forward_only \
--env MOCK_FAIL_APP_UP=once \
--env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \
>"$run_dir/forward-failure.out" 2>&1
forward_status=$?
set -e
[[ "$forward_status" -ne 0 ]] || {
echo "Forward-only test drill did not surface the startup failure." >&2
exit 1
}
grep -F 'automatic old-image restart is blocked' "$run_dir/forward-failure.out" >/dev/null
grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null
test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
"$fixture/mock-commands.log")" = 1
write_old_env
printf '%s\n' "$current_commit" >"$fixture/git-state"
: >"$fixture/mock-commands.log"
rm -f "$fixture/app-up-failed"
set +e
run_release application_safe \
--env MOCK_FAIL_APP_UP=once \
--env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \
>"$run_dir/safe-failure.out" 2>&1
safe_status=$?
set -e
[[ "$safe_status" -ne 0 ]] || {
echo "Application-safe test drill did not surface the startup failure." >&2
exit 1
}
grep -F 'restoring the previous revision' "$run_dir/safe-failure.out" >/dev/null
grep -Fx "APP_IMAGE=who-need-help:test-${current_commit:0:12}" "$fixture/.env" >/dev/null
grep -Fx "$current_commit" "$fixture/git-state" >/dev/null
test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
"$fixture/mock-commands.log")" = 2
echo "Isolated test release success/forward-only/safe-recovery drill passed."

389
scripts/test-release-remote.sh Executable file
View File

@ -0,0 +1,389 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
action=${1:-}
root=${2:-/srv/who_need_help-test}
expected_domain=${3:-test.whoneedhelp.com}
bundle=${4:-}
target_commit=${5:-}
backup=${6:-}
expected_migration_policy=${7:-}
image_archive=${8:-}
image_manifest=${9:-}
usage() {
echo "Usage: $0 plan /srv/who_need_help-test test.whoneedhelp.com" >&2
echo " $0 apply /srv/who_need_help-test test.whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2
}
case "$action" in
plan | apply) ;;
*) usage; exit 2 ;;
esac
for command in curl docker git gzip jq pg_restore realpath sha256sum; do
command -v "$command" >/dev/null 2>&1 || {
echo "Required test release command is unavailable: $command" >&2
exit 2
}
done
root=$(realpath --canonicalize-existing "$root")
[[ "$root" == /srv/who_need_help-test ]] || {
echo "Refusing a test release outside /srv/who_need_help-test." >&2
exit 2
}
env_file="$root/.env"
[[ -f "$env_file" && "$(stat -c '%a' "$env_file")" == 600 ]] || {
echo "Test .env is missing or does not have mode 0600." >&2
exit 2
}
read_value() {
local key=$1
awk -v key="$key" '
index($0, key "=") == 1 {
print substr($0, length(key) + 2)
found = 1
exit
}
END { if (!found) exit 1 }
' "$env_file"
}
deployment_environment=$(read_value DEPLOYMENT_ENV)
compose_project=$(read_value COMPOSE_PROJECT_NAME)
database_mode=$(read_value DATABASE_MODE)
app_topology=$(read_value APP_TOPOLOGY)
phx_host=$(read_value PHX_HOST)
public_origin=$(read_value WNH_BASE_URL)
current_commit=$(git -C "$root" rev-parse --verify HEAD)
[[ "$deployment_environment" == test ]] || {
echo "DEPLOYMENT_ENV is not test." >&2
exit 2
}
[[ "$compose_project" == who_need_help_test ]] || {
echo "Unexpected test Compose project." >&2
exit 2
}
[[ "$database_mode" == container ]] || {
echo "The verified test workflow expects DATABASE_MODE=container." >&2
exit 2
}
[[ "$phx_host" == "$expected_domain" &&
"$public_origin" == "https://$expected_domain" ]] || {
echo "Test origin does not match the expected domain." >&2
exit 2
}
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
echo "Test checkout has uncommitted files." >&2
exit 2
}
"$root/scripts/compose.sh" "$env_file" config --quiet
case "$app_topology" in
compact)
expected_services=(app)
runtime_services=(app)
;;
split)
expected_services=(web worker)
runtime_services=(docker-api-proxy proxy web worker)
;;
*) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;;
esac
for service in db "${expected_services[@]}"; do
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
[[ ${#containers[@]} -gt 0 ]] || {
echo "Test service is not running: $service" >&2
exit 2
}
for container in "${containers[@]}"; do
state=$(docker inspect --format '{{.State.Status}}' "$container")
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
[[ "$state" == running && "$health" == healthy ]] || {
echo "Test container is not healthy: $service" >&2
exit 2
}
done
done
curl --fail --silent --show-error --max-time 15 \
"https://$expected_domain/healthz/ready" >/dev/null
printf 'Test checkout: %s\n' "$root"
printf 'Current commit: %s\n' "$current_commit"
printf 'Compose project: %s\n' "$compose_project"
printf 'Topology: %s\n' "$app_topology"
printf 'Database mode: %s\n' "$database_mode"
printf 'Public readiness: passed\n'
df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root"
if [[ "$action" == plan ]]; then
echo "Read-only test release scope check passed."
exit 0
fi
if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
-z "$expected_migration_policy" || -z "$image_archive" ||
-z "$image_manifest" ]]; then
usage
exit 2
fi
expected_confirmation="$expected_domain:$target_commit"
[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$expected_confirmation" ]] || {
echo "Set WNH_TEST_RELEASE_CONFIRM=$expected_confirmation for the approved test release." >&2
exit 2
}
for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
"$image_archive" "$image_archive.sha256" "$image_manifest"; do
[[ -f "$required_file" ]] || {
echo "Required test release evidence is missing: $required_file" >&2
exit 2
}
done
(
cd "$(dirname -- "$bundle")"
sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null
)
(
cd "$(dirname -- "$backup")"
sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
)
pg_restore --list "$backup" >/dev/null
(
cd "$(dirname -- "$image_archive")"
sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null
)
gzip -t "$image_archive"
grep -Fx 'format=1' "$image_manifest" >/dev/null
grep -Fx 'deployment=test' "$image_manifest" >/dev/null
grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
git -C "$root" bundle verify "$bundle" >/dev/null
bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
[[ "$bundle_head" == "$target_commit" ]] || {
echo "Bundle HEAD does not match the approved test commit." >&2
exit 2
}
release_ref="refs/wnh/test-releases/$target_commit"
git -C "$root" fetch "$bundle" "HEAD:$release_ref"
[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || {
echo "Fetched test release ref does not match the approved commit." >&2
exit 1
}
git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || {
echo "Test updates must be a fast-forward from the deployed commit." >&2
exit 1
}
policy_script=$(mktemp)
trap 'rm -f "$policy_script"' EXIT HUP INT TERM
git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script"
chmod 700 "$policy_script"
migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root")
rm -f "$policy_script"
trap - EXIT HUP INT TERM
printf '%s\n' "$migration_policy_output"
migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output")
[[ "$migration_policy" == "$expected_migration_policy" ]] || {
echo "Remote migration policy does not match the locally approved policy." >&2
exit 2
}
if [[ "$migration_policy" == forward_only ]]; then
forward_confirmation="$expected_domain:$target_commit:forward-only"
[[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
echo "Set WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation for this test schema boundary." >&2
exit 2
}
fi
release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}"
release_dir="$root/output/releases/$release_id"
mkdir -p "$release_dir"
chmod 700 "$root/output" "$root/output/releases" "$release_dir"
rollback_manifest="$release_dir/rollback-manifest.txt"
{
printf 'previous_commit=%s\n' "$current_commit"
printf 'target_commit=%s\n' "$target_commit"
printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)"
printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)"
printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)"
printf 'migration_policy=%s\n' "$migration_policy"
printf 'database_backup=%s\n' "$backup"
printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf 'status=started\n'
} >"$rollback_manifest"
chmod 600 "$rollback_manifest"
revision_changed=false
migration_started=false
restore_previous_revision() {
local temporary
temporary=$(mktemp "$root/.env.test-release-rollback.XXXXXX")
chmod 600 "$temporary"
APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
awk '
BEGIN {
replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
}
{
separator = index($0, "=")
key = separator > 1 ? substr($0, 1, separator - 1) : ""
print (key in replacement) ? key "=" replacement[key] : $0
}
' "$env_file" >"$temporary"
mv "$temporary" "$env_file"
chmod 600 "$env_file"
git -C "$root" checkout --detach "$current_commit" >/dev/null
}
rollback_runtime() {
local status=$?
trap - EXIT HUP INT TERM
if [[ "$status" -ne 0 && "$revision_changed" == true &&
"$migration_policy" == forward_only && "$migration_started" == true ]]; then
{
printf 'status=forward-only-release-failed\n'
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf 'automatic_application_rollback=blocked\n'
} >>"$rollback_manifest"
echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2
elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then
echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2
restore_previous_revision
"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db || true
"$root/scripts/compose.sh" "$env_file" \
up -d --no-deps --no-build --force-recreate --wait \
"${runtime_services[@]}" || true
{
printf 'status=runtime-rolled-back\n'
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} >>"$rollback_manifest"
fi
exit "$status"
}
trap rollback_runtime EXIT HUP INT TERM
gzip -dc "$image_archive" | docker load >/dev/null
git -C "$root" checkout --detach "$release_ref" >/dev/null
"$root/scripts/set-deployment-revision.sh" "$env_file"
revision_changed=true
"$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain"
expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)")
if [[ "$app_topology" == split ]]; then
expected_images+=(
"$(read_value SOCKET_PROXY_IMAGE)"
"$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')"
)
fi
grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null
test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}"
for image in "${expected_images[@]}"; do
expected_id=$(awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' "$image_manifest")
[[ -n "$expected_id" ]] || {
echo "Image manifest is missing the expected image: $image" >&2
exit 2
}
[[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || {
echo "Loaded test image ID does not match the manifest: $image" >&2
exit 2
}
[[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || {
echo "Loaded test image is not linux/amd64: $image" >&2
exit 2
}
done
"$root/scripts/restore-drill-compose.sh" "$backup"
if [[ "$migration_policy" == forward_only ]]; then
echo "Stopping the old test application before the forward-only migration boundary."
"$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}"
fi
"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db
verify_service_image() {
local service=$1 expected_image=$2 require_health=$3
local expected_image_id container state health configured_image running_image_id
expected_image_id=$(docker image inspect --format '{{.Id}}' "$expected_image")
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
[[ ${#containers[@]} -gt 0 ]] || {
echo "Candidate test service has no container: $service" >&2
return 1
}
for container in "${containers[@]}"; do
state=$(docker inspect --format '{{.State.Status}}' "$container")
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
configured_image=$(docker inspect --format '{{.Config.Image}}' "$container")
running_image_id=$(docker inspect --format '{{.Image}}' "$container")
[[ "$state" == running ]] || {
echo "Candidate test container is not running: $service" >&2
return 1
}
if [[ "$require_health" == true && "$health" != healthy ]]; then
echo "Candidate test container is not healthy: $service" >&2
return 1
fi
[[ "$configured_image" == "$expected_image" &&
"$running_image_id" == "$expected_image_id" ]] || {
echo "Candidate test service does not use its approved image: $service" >&2
return 1
}
done
}
verify_service_image db "$(read_value POSTGIS_IMAGE)" true
migration_started=true
"$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate
"$root/scripts/check-database.sh" "$env_file" </dev/null
"$root/scripts/compose.sh" "$env_file" \
up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}"
expected_app_image=$(read_value APP_IMAGE)
for service in "${expected_services[@]}"; do
verify_service_image "$service" "$expected_app_image" true
done
if [[ "$app_topology" == split ]]; then
verify_service_image docker-api-proxy "$(read_value SOCKET_PROXY_IMAGE)" false
verify_service_image proxy \
"$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" \
false
fi
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-realtime-cluster.sh" compose
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-beam-runtime.sh" compose
curl --fail --silent --show-error --max-time 30 \
"https://$expected_domain/healthz/ready" >/dev/null
{
printf 'status=success\n'
printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} >>"$rollback_manifest"
revision_changed=false
trap - EXIT HUP INT TERM
printf 'Test release completed: %s\n' "$target_commit"
printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
printf 'Database backup: %s\n' "$backup"

192
scripts/test-release.sh Executable file
View File

@ -0,0 +1,192 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
action=${1:-plan}
ssh_target=${2:-whoneedhelp}
remote_root=${WNH_TEST_REMOTE_ROOT:-/srv/who_need_help-test}
production_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production}
expected_domain=${WNH_TEST_DOMAIN:-test.whoneedhelp.com}
case "$action" in
plan | prepare | apply) ;;
*)
echo "Usage: $0 [plan|prepare|apply] [SSH_TARGET]" >&2
exit 2
;;
esac
for command in git gzip mktemp pg_restore realpath scp sha256sum ssh; do
command -v "$command" >/dev/null 2>&1 || {
echo "Required command is unavailable: $command" >&2
exit 2
}
done
local_commit=$(git -C "$ROOT" rev-parse --verify HEAD)
remote_commit=$(
ssh -o BatchMode=yes "$ssh_target" \
"git -C '$remote_root' rev-parse --verify HEAD"
)
printf 'Local candidate commit: %s\n' "$local_commit"
printf 'Current test commit: %s\n' "$remote_commit"
git -C "$ROOT" cat-file -e "$remote_commit^{commit}" 2>/dev/null || {
echo "The test commit is not present in the local object database." >&2
exit 2
}
git -C "$ROOT" merge-base --is-ancestor "$remote_commit" "$local_commit" || {
echo "The local candidate is not a fast-forward from the test commit." >&2
exit 2
}
printf 'Pending commits: %s\n' \
"$(git -C "$ROOT" rev-list --count "$remote_commit..$local_commit")"
legacy_edge_paths=(compose.edge.yaml deploy/caddy/Caddyfile)
if ! git -C "$ROOT" diff --quiet \
"$remote_commit" "$local_commit" -- "${legacy_edge_paths[@]}"; then
echo "The test application release contains shared edge routing changes." >&2
echo "Move route changes through the independent server-edge workflow." >&2
exit 2
fi
echo "Shared edge routing files are unchanged; the test release will not manage Caddy."
migration_policy_output=$(
"$ROOT/scripts/release-migration-policy.sh" "$remote_commit" "$local_commit"
)
printf '%s\n' "$migration_policy_output"
migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output")
plan_failed=0
if ! ssh -o BatchMode=yes "$ssh_target" \
"cd '$remote_root' && ./scripts/validate-test-env.sh .env '$expected_domain'"; then
plan_failed=1
fi
if ! ssh -o BatchMode=yes "$ssh_target" \
"cd '$remote_root' && ./scripts/validate-deployment-isolation.sh '$remote_root' '$production_root'"; then
plan_failed=1
fi
if ! ssh -o BatchMode=yes "$ssh_target" \
"bash -s -- plan '$remote_root' '$expected_domain'" \
<"$ROOT/scripts/test-release-remote.sh"; then
plan_failed=1
fi
if ! ssh -o BatchMode=yes "$ssh_target" \
"cd '$remote_root' && ./scripts/check-environment-readiness.sh .env --require-server-release"; then
plan_failed=1
fi
if [[ "$plan_failed" -ne 0 ]]; then
echo "Test release plan has blocking checks; no remote state was changed." >&2
exit 1
fi
if [[ "$action" == plan ]]; then
echo "Test release plan passed; no remote state was changed."
exit 0
fi
[[ -z "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]] || {
echo "Refusing to release a dirty checkout." >&2
exit 2
}
artifact_root=${WNH_TEST_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/test-releases"}
case "$artifact_root" in
/*) ;;
*)
echo "WNH_TEST_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
exit 2
;;
esac
mkdir -p "$artifact_root"
artifact_root=$(realpath --canonicalize-existing "$artifact_root")
WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT="$artifact_root" \
"$ROOT/scripts/prepare-production-release.sh"
release_dir="$artifact_root/$local_commit"
bundle="$release_dir/who_need_help-$local_commit.bundle"
image_archive="$release_dir/who_need_help-$local_commit-test-images-linux-amd64.tar.gz"
image_checksum="$image_archive.sha256"
image_manifest="$release_dir/who_need_help-$local_commit-test-images.manifest"
test_env=$(mktemp)
cleanup_test_env() {
trap - EXIT HUP INT TERM
rm -f "$test_env"
}
trap cleanup_test_env EXIT HUP INT TERM
scp -p "$ssh_target:$remote_root/.env" "$test_env"
chmod 600 "$test_env"
WNH_TEST_RELEASE_ARTIFACT_ROOT="$artifact_root" \
"$ROOT/scripts/prepare-test-images.sh" "$test_env"
cleanup_test_env
if [[ "$action" == prepare ]]; then
echo "Test release artifacts are prepared and verified locally; no remote state was changed."
exit 0
fi
remote_main=$(git -C "$ROOT" ls-remote origin refs/heads/main | awk '{print $1}')
[[ "$remote_main" == "$local_commit" ]] || {
echo "origin/main does not contain the exact approved test candidate." >&2
echo "Expected: $local_commit" >&2
echo "Observed: ${remote_main:-missing}" >&2
exit 2
}
confirmation="$expected_domain:$local_commit"
[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$confirmation" ]] || {
echo "Test release execution requires exact approval:" >&2
echo "WNH_TEST_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2
exit 2
}
if [[ "$migration_policy" == forward_only ]]; then
forward_confirmation="$expected_domain:$local_commit:forward-only"
[[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
echo "This test release contains forward-only migrations." >&2
echo "WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation \\" >&2
echo " WNH_TEST_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2
exit 2
}
fi
remote_release_dir="$remote_root/output/releases/incoming"
remote_bundle="$remote_release_dir/$(basename -- "$bundle")"
remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")"
remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")"
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
remote_backup="$remote_root/output/backups/test/pre-$timestamp-${local_commit:0:12}.dump"
ssh -o BatchMode=yes "$ssh_target" "install -d -m 700 '$remote_release_dir'"
scp -p \
"$bundle" "$bundle.sha256" \
"$image_archive" "$image_checksum" "$image_manifest" \
"$ssh_target:$remote_release_dir/"
ssh -o BatchMode=yes "$ssh_target" \
"cd '$remote_root' && ./scripts/backup-compose.sh '$remote_backup'"
local_backup_dir="$ROOT/output/test-backups/$timestamp-${local_commit:0:12}"
mkdir -p "$local_backup_dir"
chmod 700 "$ROOT/output" "$ROOT/output/test-backups" "$local_backup_dir"
scp -p \
"$ssh_target:$remote_backup" \
"$ssh_target:$remote_backup.sha256" \
"$local_backup_dir/"
(
cd "$local_backup_dir"
sha256sum --check "$(basename -- "$remote_backup.sha256")" >/dev/null
)
pg_restore --list "$local_backup_dir/$(basename -- "$remote_backup")" >/dev/null
echo "Copied and independently verified the pre-release test backup outside the server."
quoted_confirmation=$(printf '%q' "$confirmation")
quoted_forward_confirmation=$(printf '%q' "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}")
ssh -o BatchMode=yes "$ssh_target" \
"WNH_TEST_RELEASE_CONFIRM=$quoted_confirmation WNH_TEST_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \
<"$ROOT/scripts/test-release-remote.sh"
echo "Test release and public health verification completed."

15
test/scripts/production_release_artifact_root_test.py
View File

@ -91,6 +91,21 @@ class ProductionReleaseArtifactRootTest(unittest.TestCase):
self.assertEqual(result.returncode, 2)
self.assertIn("must be an absolute path", result.stderr)
def test_untracked_build_input_is_rejected(self):
(self.project / "untracked-build-input.txt").write_text(
"must not enter an immutable release\n", encoding="utf-8"
)
result = self.run_command(
[str(self.scripts / "prepare-production-release.sh")],
cwd=self.project,
env=self.artifact_env(),
check=False,
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("dirty checkout", result.stderr)
def test_bundle_manifest_for_another_commit_is_rejected(self):
self.run_command(
[str(self.scripts / "prepare-production-release.sh")],

178
test/scripts/test_release_artifact_root_test.py Normal file
View File

@ -0,0 +1,178 @@
import gzip
import hashlib
import os
import shutil
import subprocess
import tempfile
import textwrap
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
class TestReleaseArtifactRootTest(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.TemporaryDirectory()
self.base = Path(self.tempdir.name)
self.project = self.base / "project"
self.scripts = self.project / "scripts"
self.scripts.mkdir(parents=True)
script = self.scripts / "prepare-test-images.sh"
shutil.copy2(ROOT / "scripts" / script.name, script)
script.chmod(0o755)
self.run_command(["git", "init", "--quiet"], cwd=self.project)
self.run_command(
["git", "config", "user.email", "test-release@example.invalid"],
cwd=self.project,
)
self.run_command(
["git", "config", "user.name", "Test release"], cwd=self.project
)
self.run_command(["git", "add", "scripts"], cwd=self.project)
self.run_command(
["git", "commit", "--quiet", "-m", "test fixture"], cwd=self.project
)
self.commit = self.run_command(
["git", "rev-parse", "HEAD"], cwd=self.project
).stdout.strip()
self.artifact_root = self.base / "test-artifacts"
self.fake_bin = self.base / "bin"
self.fake_bin.mkdir()
for name in ("docker", "jq"):
command = self.fake_bin / name
command.write_text("#!/bin/sh\nexit 97\n", encoding="utf-8")
command.chmod(0o755)
self.test_env = self.base / "test.env"
self.test_env.write_text(
textwrap.dedent(
"""\
DEPLOYMENT_ENV=test
DATABASE_MODE=container
APP_TOPOLOGY=compact
APP_IMAGE=replace-me
SOCKET_PROXY_IMAGE=replace-me
POSTGIS_IMAGE=replace-me
"""
),
encoding="utf-8",
)
self.test_env.chmod(0o600)
def tearDown(self):
self.tempdir.cleanup()
def run_command(self, command, *, cwd=None, env=None, check=True):
return subprocess.run(
command,
cwd=cwd,
env=env,
capture_output=True,
text=True,
check=check,
)
def environment(self):
env = os.environ.copy()
env["WNH_TEST_RELEASE_ARTIFACT_ROOT"] = str(self.artifact_root)
env["PATH"] = f"{self.fake_bin}:{env['PATH']}"
return env
def write_existing_artifact(self):
release_dir = self.artifact_root / self.commit
release_dir.mkdir(parents=True)
archive = (
release_dir
/ f"who_need_help-{self.commit}-test-images-linux-amd64.tar.gz"
)
with archive.open("wb") as output:
with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed:
compressed.write(b"verified test image archive fixture")
archive.chmod(0o600)
digest = hashlib.sha256(archive.read_bytes()).hexdigest()
checksum = Path(f"{archive}.sha256")
checksum.write_text(f"{digest} {archive.name}\n", encoding="utf-8")
checksum.chmod(0o600)
short_commit = self.commit[:12]
manifest = release_dir / f"who_need_help-{self.commit}-test-images.manifest"
manifest.write_text(
textwrap.dedent(
f"""\
format=1
deployment=test
commit={self.commit}
platform=linux/amd64
topology=compact
image_count=2
image=who-need-help:test-{short_commit}|sha256:{'a' * 64}
image=who-need-help:postgis-test-{short_commit}|sha256:{'b' * 64}
"""
),
encoding="utf-8",
)
manifest.chmod(0o600)
return archive, manifest
def test_existing_test_archive_is_verified_without_building(self):
archive, manifest = self.write_existing_artifact()
result = self.run_command(
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
cwd=self.project,
env=self.environment(),
)
self.assertIn("verifying all metadata", result.stdout)
self.assertIn(str(archive), result.stdout)
manifest.write_text(
manifest.read_text(encoding="utf-8").replace(
"deployment=test", "deployment=production"
),
encoding="utf-8",
)
rejected = self.run_command(
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
cwd=self.project,
env=self.environment(),
check=False,
)
self.assertEqual(rejected.returncode, 2)
self.assertIn("wrong deployment identity", rejected.stderr)
def test_relative_artifact_root_is_rejected(self):
env = self.environment()
env["WNH_TEST_RELEASE_ARTIFACT_ROOT"] = "relative/test-releases"
result = self.run_command(
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
cwd=self.project,
env=env,
check=False,
)
self.assertEqual(result.returncode, 2)
self.assertIn("must be an absolute path", result.stderr)
def test_untracked_build_input_is_rejected(self):
(self.project / "untracked-build-input.txt").write_text(
"must not enter an immutable release\n", encoding="utf-8"
)
result = self.run_command(
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
cwd=self.project,
env=self.environment(),
check=False,
)
self.assertEqual(result.returncode, 2)
self.assertIn("dirty checkout", result.stderr)
if __name__ == "__main__":
unittest.main()