Add isolated public test release workflow
This commit is contained in:
parent
86260cead1
commit
4d09caf130
|
|
@ -361,14 +361,32 @@ TEST_GOOGLE_OAUTH_AUTHORIZED_PARTY_IDS=YOUR_TEST_ANDROID_CLIENT_ID \
|
||||||
./scripts/deploy-up.sh .env
|
./scripts/deploy-up.sh .env
|
||||||
```
|
```
|
||||||
|
|
||||||
For later test updates, check out the desired clean revision and update only
|
For later public test updates, release the exact clean revision with immutable
|
||||||
the image tags. Existing deployment secrets remain unchanged:
|
images built on the operator workstation. The test release workflow performs
|
||||||
|
a read-only scope plan, builds and verifies `linux/amd64` images outside the
|
||||||
|
4-GiB server, creates and copies an independently verified database backup,
|
||||||
|
runs a restore-and-migrate drill, and starts Compose with `--no-build`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./scripts/set-deployment-revision.sh .env
|
./scripts/test-release.sh plan whoneedhelp
|
||||||
./scripts/deploy-up.sh .env
|
./scripts/test-release.sh prepare whoneedhelp
|
||||||
|
|
||||||
|
# Make the selected commit available as origin/main before apply. The apply
|
||||||
|
# gate verifies that origin/main equals the exact local SHA.
|
||||||
|
candidate=$(git rev-parse HEAD)
|
||||||
|
WNH_TEST_RELEASE_CONFIRM="test.whoneedhelp.com:$candidate" \
|
||||||
|
WNH_TEST_FORWARD_ONLY_CONFIRM="test.whoneedhelp.com:$candidate:forward-only" \
|
||||||
|
./scripts/test-release.sh apply whoneedhelp
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Omit `WNH_TEST_FORWARD_ONLY_CONFIRM` when the read-only plan reports
|
||||||
|
`migration_policy=application_safe`. The workflow refuses shared Caddy changes,
|
||||||
|
requires a fast-forward from the deployed test commit, preserves the single
|
||||||
|
test `.env`, and never addresses the production Compose project or database.
|
||||||
|
Do not use `deploy-up.sh` for an ordinary public test update on the small
|
||||||
|
server: that command intentionally includes `--build` and therefore builds the
|
||||||
|
release on the target host.
|
||||||
|
|
||||||
Test always uses its own `who_need_help_test` PostGIS container/volume. Local
|
Test always uses its own `who_need_help_test` PostGIS container/volume. Local
|
||||||
development can use Mailpit; a public test deployment must use its own SMTP
|
development can use Mailpit; a public test deployment must use its own SMTP
|
||||||
password and a visibly test-specific sender identity.
|
password and a visibly test-specific sender identity.
|
||||||
|
|
|
||||||
|
|
@ -17,8 +17,8 @@ for command in docker gzip jq sha256sum; do
|
||||||
}
|
}
|
||||||
done
|
done
|
||||||
|
|
||||||
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
|
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
|
||||||
echo "Refusing to build production images from a dirty tracked checkout." >&2
|
echo "Refusing to build production images from a dirty checkout." >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,8 @@ umask 077
|
||||||
|
|
||||||
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
|
||||||
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
|
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
|
||||||
echo "Refusing to package a release from a dirty tracked checkout." >&2
|
echo "Refusing to package a release from a dirty checkout." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
221
scripts/prepare-test-images.sh
Executable file
221
scripts/prepare-test-images.sh
Executable file
|
|
@ -0,0 +1,221 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
source_env=${1:-}
|
||||||
|
|
||||||
|
if [[ -z "$source_env" || ! -f "$source_env" ]]; then
|
||||||
|
echo "Usage: $0 TEST_ENV_FILE" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
for command in docker gzip jq realpath sha256sum; do
|
||||||
|
command -v "$command" >/dev/null 2>&1 || {
|
||||||
|
echo "Required command is unavailable: $command" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
|
||||||
|
echo "Refusing to build test images from a dirty checkout." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
commit=$(git -C "$ROOT" rev-parse --verify HEAD)
|
||||||
|
short_commit=${commit:0:12}
|
||||||
|
artifact_root=${WNH_TEST_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/test-releases"}
|
||||||
|
case "$artifact_root" in
|
||||||
|
/*) ;;
|
||||||
|
*)
|
||||||
|
echo "WNH_TEST_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
mkdir -p "$artifact_root"
|
||||||
|
artifact_root=$(realpath --canonicalize-existing "$artifact_root")
|
||||||
|
release_dir="$artifact_root/$commit"
|
||||||
|
archive="$release_dir/who_need_help-$commit-test-images-linux-amd64.tar.gz"
|
||||||
|
checksum="$archive.sha256"
|
||||||
|
manifest="$release_dir/who_need_help-$commit-test-images.manifest"
|
||||||
|
|
||||||
|
mkdir -p "$release_dir"
|
||||||
|
chmod 700 "$artifact_root" "$release_dir"
|
||||||
|
|
||||||
|
build_env=$(mktemp "$release_dir/.test-image-build.XXXXXX")
|
||||||
|
cleanup() {
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
rm -f "$build_env"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT HUP INT TERM
|
||||||
|
install -m 600 "$source_env" "$build_env"
|
||||||
|
|
||||||
|
read_value() {
|
||||||
|
local key=$1
|
||||||
|
awk -v key="$key" '
|
||||||
|
index($0, key "=") == 1 {
|
||||||
|
print substr($0, length(key) + 2)
|
||||||
|
found = 1
|
||||||
|
exit
|
||||||
|
}
|
||||||
|
END { if (!found) exit 1 }
|
||||||
|
' "$build_env"
|
||||||
|
}
|
||||||
|
|
||||||
|
replace_value() {
|
||||||
|
local key=$1 value=$2 temporary
|
||||||
|
temporary=$(mktemp "$release_dir/.test-image-env.XXXXXX")
|
||||||
|
chmod 600 "$temporary"
|
||||||
|
awk -v key="$key" -v value="$value" '
|
||||||
|
index($0, key "=") == 1 { print key "=" value; found = 1; next }
|
||||||
|
{ print }
|
||||||
|
END { if (!found) exit 1 }
|
||||||
|
' "$build_env" >"$temporary"
|
||||||
|
mv "$temporary" "$build_env"
|
||||||
|
chmod 600 "$build_env"
|
||||||
|
}
|
||||||
|
|
||||||
|
[[ "$(read_value DEPLOYMENT_ENV)" == test ]] || {
|
||||||
|
echo "The image build input is not a test environment." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(read_value DATABASE_MODE)" == container ]] || {
|
||||||
|
echo "The verified test image workflow expects DATABASE_MODE=container." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
replace_value APP_IMAGE "who-need-help:test-$short_commit"
|
||||||
|
replace_value SOCKET_PROXY_IMAGE \
|
||||||
|
"who-need-help:socket-proxy-test-$short_commit"
|
||||||
|
replace_value POSTGIS_IMAGE "who-need-help:postgis-test-$short_commit"
|
||||||
|
|
||||||
|
topology=$(read_value APP_TOPOLOGY)
|
||||||
|
case "$topology" in
|
||||||
|
compact)
|
||||||
|
build_services=(migrate db)
|
||||||
|
;;
|
||||||
|
split)
|
||||||
|
build_services=(docker-api-proxy proxy migrate db)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "APP_TOPOLOGY must be compact or split." >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
app_image=$(read_value APP_IMAGE)
|
||||||
|
postgis_image=$(read_value POSTGIS_IMAGE)
|
||||||
|
images=("$app_image" "$postgis_image")
|
||||||
|
if [[ "$topology" == split ]]; then
|
||||||
|
socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE)
|
||||||
|
proxy_image=$(
|
||||||
|
"$ROOT/scripts/compose.sh" "$build_env" config --format json |
|
||||||
|
jq -er '.services.proxy.image'
|
||||||
|
)
|
||||||
|
images+=("$socket_proxy_image" "$proxy_image")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then
|
||||||
|
[[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || {
|
||||||
|
echo "The test image package is incomplete; refusing to overwrite it." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
(
|
||||||
|
cd "$release_dir"
|
||||||
|
sha256sum --check "$(basename -- "$checksum")" >/dev/null
|
||||||
|
)
|
||||||
|
echo "Test image package already exists; verifying all metadata."
|
||||||
|
else
|
||||||
|
"$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}"
|
||||||
|
|
||||||
|
manifest_tmp=$(mktemp "$release_dir/.test-images-manifest.XXXXXX")
|
||||||
|
archive_tmp=$(mktemp "$release_dir/.test-images-archive.XXXXXX")
|
||||||
|
trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'format=1\n'
|
||||||
|
printf 'deployment=test\n'
|
||||||
|
printf 'commit=%s\n' "$commit"
|
||||||
|
printf 'platform=linux/amd64\n'
|
||||||
|
printf 'topology=%s\n' "$topology"
|
||||||
|
printf 'image_count=%s\n' "${#images[@]}"
|
||||||
|
for image in "${images[@]}"; do
|
||||||
|
platform=$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")
|
||||||
|
[[ "$platform" == linux/amd64 ]] || {
|
||||||
|
echo "Test image has an unexpected platform: $image ($platform)" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
image_id=$(docker image inspect --format '{{.Id}}' "$image")
|
||||||
|
printf 'image=%s|%s\n' "$image" "$image_id"
|
||||||
|
done
|
||||||
|
} >"$manifest_tmp"
|
||||||
|
|
||||||
|
docker save "${images[@]}" | gzip -n -9 >"$archive_tmp"
|
||||||
|
mv "$archive_tmp" "$archive"
|
||||||
|
mv "$manifest_tmp" "$manifest"
|
||||||
|
chmod 600 "$archive" "$manifest"
|
||||||
|
hash=$(sha256sum "$archive" | awk '{print $1}')
|
||||||
|
printf '%s %s\n' "$hash" "$(basename -- "$archive")" >"$checksum"
|
||||||
|
chmod 600 "$checksum"
|
||||||
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$release_dir"
|
||||||
|
sha256sum --check "$(basename -- "$checksum")" >/dev/null
|
||||||
|
)
|
||||||
|
gzip -t "$archive"
|
||||||
|
|
||||||
|
manifest_value() {
|
||||||
|
local key=$1
|
||||||
|
awk -F= -v key="$key" '
|
||||||
|
$1 == key { count += 1; value = substr($0, length(key) + 2) }
|
||||||
|
END {
|
||||||
|
if (count != 1) exit 1
|
||||||
|
print value
|
||||||
|
}
|
||||||
|
' "$manifest"
|
||||||
|
}
|
||||||
|
|
||||||
|
[[ "$(manifest_value format)" == 1 ]] || {
|
||||||
|
echo "Test image manifest format is unsupported." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(manifest_value deployment)" == test ]] || {
|
||||||
|
echo "Test image manifest has the wrong deployment identity." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(manifest_value commit)" == "$commit" ]] || {
|
||||||
|
echo "Test image manifest commit does not match the current commit." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(manifest_value platform)" == linux/amd64 ]] || {
|
||||||
|
echo "Test image manifest platform is not linux/amd64." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(manifest_value topology)" == "$topology" ]] || {
|
||||||
|
echo "Test image manifest topology does not match the environment." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(manifest_value image_count)" == "${#images[@]}" ]] || {
|
||||||
|
echo "Test image manifest count does not match the required images." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
test "$(grep -c '^image=' "$manifest")" = "${#images[@]}"
|
||||||
|
for image in "${images[@]}"; do
|
||||||
|
awk -F'|' -v image="$image" '
|
||||||
|
$1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 }
|
||||||
|
END { if (!found) exit 1 }
|
||||||
|
' "$manifest"
|
||||||
|
done
|
||||||
|
|
||||||
|
archive_hash=$(sha256sum "$archive" | awk '{print $1}')
|
||||||
|
expected_checksum="$archive_hash $(basename -- "$archive")"
|
||||||
|
[[ "$(cat -- "$checksum")" == "$expected_checksum" ]] || {
|
||||||
|
echo "Test image checksum metadata does not name the exact archive." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup
|
||||||
|
printf 'Test image archive: %s\n' "$archive"
|
||||||
|
printf 'Image archive checksum: %s\n' "$checksum"
|
||||||
|
printf 'Image manifest: %s\n' "$manifest"
|
||||||
|
|
@ -157,7 +157,7 @@ cat >"$mock_bin/git" <<'EOF'
|
||||||
set -eu
|
set -eu
|
||||||
case " $* " in
|
case " $* " in
|
||||||
*' symbolic-ref --quiet --short HEAD '*) exit 1 ;;
|
*' symbolic-ref --quiet --short HEAD '*) exit 1 ;;
|
||||||
*' status --porcelain --untracked-files=no '*) exit 0 ;;
|
*' status --porcelain --untracked-files=normal '*) exit 0 ;;
|
||||||
*' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;;
|
*' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;;
|
||||||
*' rev-parse refs/wnh/releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
|
*' rev-parse refs/wnh/releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
|
||||||
*' bundle verify '*) exit 0 ;;
|
*' bundle verify '*) exit 0 ;;
|
||||||
|
|
|
||||||
|
|
@ -83,8 +83,8 @@ current_commit=$(git -C "$root" rev-parse --verify HEAD)
|
||||||
echo "Production checkout must be on main or detached at the deployed commit." >&2
|
echo "Production checkout must be on main or detached at the deployed commit." >&2
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
[[ -z "$(git -C "$root" status --porcelain --untracked-files=no)" ]] || {
|
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
|
||||||
echo "Production checkout has tracked modifications." >&2
|
echo "Production checkout has uncommitted files." >&2
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -100,8 +100,8 @@ if [[ "$action" == "plan" ]]; then
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then
|
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
|
||||||
echo "Refusing to release a dirty tracked checkout." >&2
|
echo "Refusing to release a dirty checkout." >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -144,6 +144,7 @@ echo "Checking release migration compatibility policy"
|
||||||
|
|
||||||
echo "Checking isolated production release orchestration"
|
echo "Checking isolated production release orchestration"
|
||||||
./scripts/production-release-drill.sh
|
./scripts/production-release-drill.sh
|
||||||
|
./scripts/test-release-drill.sh
|
||||||
|
|
||||||
echo "Checking Dockerfiles with Hadolint 2.14.0"
|
echo "Checking Dockerfiles with Hadolint 2.14.0"
|
||||||
for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \
|
for dockerfile in Dockerfile Dockerfile.backup Dockerfile.minio \
|
||||||
|
|
@ -1598,6 +1599,7 @@ docker run --rm \
|
||||||
"$python_runtime_image" python test/scripts/install_production_external_monitor_test.py
|
"$python_runtime_image" python test/scripts/install_production_external_monitor_test.py
|
||||||
python3 test/scripts/production_release_artifact_root_test.py
|
python3 test/scripts/production_release_artifact_root_test.py
|
||||||
python3 test/scripts/production_release_clean_test.py
|
python3 test/scripts/production_release_clean_test.py
|
||||||
|
python3 test/scripts/test_release_artifact_root_test.py
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
--volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \
|
--volume "$ROOT/ops/external-boundaries/mock_server.py:/src/mock_server.py:ro" \
|
||||||
"$python_runtime_image" python -c \
|
"$python_runtime_image" python -c \
|
||||||
|
|
|
||||||
|
|
@ -29,8 +29,8 @@ if [ "$(stat -c '%a' "$env_file")" != 600 ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]; then
|
if [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]; then
|
||||||
echo "Refusing to select a deployment revision from a dirty tracked checkout." >&2
|
echo "Refusing to select a deployment revision from a dirty checkout." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
358
scripts/test-release-drill.sh
Executable file
358
scripts/test-release-drill.sh
Executable file
|
|
@ -0,0 +1,358 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
BASE_IMAGE="debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd"
|
||||||
|
mkdir -p "$ROOT/output"
|
||||||
|
run_dir=$(mktemp -d "$ROOT/output/test-release-drill.XXXXXX")
|
||||||
|
fixture="$run_dir/test"
|
||||||
|
mock_bin="$run_dir/mock-bin"
|
||||||
|
remote_root=/srv/who_need_help-test
|
||||||
|
current_commit=1111111111111111111111111111111111111111
|
||||||
|
target_commit=2222222222222222222222222222222222222222
|
||||||
|
release_confirmation="test.whoneedhelp.com:$target_commit"
|
||||||
|
forward_confirmation="test.whoneedhelp.com:$target_commit:forward-only"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
status=$?
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
if [[ "$status" -ne 0 ]]; then
|
||||||
|
for output in "$run_dir"/*.out; do
|
||||||
|
[[ -f "$output" ]] || continue
|
||||||
|
printf '\n--- %s ---\n' "$(basename -- "$output")" >&2
|
||||||
|
sed -n '1,240p' "$output" >&2
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
find "$run_dir" -xdev -depth -delete 2>/dev/null || true
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT HUP INT TERM
|
||||||
|
|
||||||
|
install -d -m 700 \
|
||||||
|
"$fixture/.git" \
|
||||||
|
"$fixture/scripts" \
|
||||||
|
"$fixture/output/releases/incoming" \
|
||||||
|
"$fixture/output/backups/test" \
|
||||||
|
"$mock_bin"
|
||||||
|
|
||||||
|
write_old_env() {
|
||||||
|
install -m 600 /dev/null "$fixture/.env"
|
||||||
|
printf '%s\n' \
|
||||||
|
'DEPLOYMENT_ENV=test' \
|
||||||
|
'COMPOSE_PROJECT_NAME=who_need_help_test' \
|
||||||
|
'DATABASE_MODE=container' \
|
||||||
|
'APP_TOPOLOGY=compact' \
|
||||||
|
'PHX_HOST=test.whoneedhelp.com' \
|
||||||
|
'WNH_BASE_URL=https://test.whoneedhelp.com' \
|
||||||
|
"APP_IMAGE=who-need-help:test-${current_commit:0:12}" \
|
||||||
|
"SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-${current_commit:0:12}" \
|
||||||
|
"POSTGIS_IMAGE=who-need-help:postgis-test-${current_commit:0:12}" \
|
||||||
|
>"$fixture/.env"
|
||||||
|
}
|
||||||
|
write_old_env
|
||||||
|
|
||||||
|
bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle"
|
||||||
|
printf 'isolated test release drill bundle\n' >"$bundle"
|
||||||
|
bundle_hash=$(sha256sum "$bundle" | awk '{print $1}')
|
||||||
|
printf '%s %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256"
|
||||||
|
|
||||||
|
image_archive="$fixture/output/releases/incoming/who_need_help-$target_commit-test-images-linux-amd64.tar.gz"
|
||||||
|
printf 'isolated test release drill image archive\n' | gzip -n >"$image_archive"
|
||||||
|
image_hash=$(sha256sum "$image_archive" | awk '{print $1}')
|
||||||
|
printf '%s %s\n' "$image_hash" "$(basename -- "$image_archive")" \
|
||||||
|
>"$image_archive.sha256"
|
||||||
|
|
||||||
|
app_config="$run_dir/app-image-config.json"
|
||||||
|
db_config="$run_dir/db-image-config.json"
|
||||||
|
printf '%s\n' \
|
||||||
|
'{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}' \
|
||||||
|
>"$app_config"
|
||||||
|
printf '%s\n' \
|
||||||
|
'{"architecture":"amd64","os":"linux","variant":"test-db","rootfs":{"type":"layers","diff_ids":[]}}' \
|
||||||
|
>"$db_config"
|
||||||
|
app_image_id="sha256:$(sha256sum "$app_config" | awk '{print $1}')"
|
||||||
|
db_image_id="sha256:$(sha256sum "$db_config" | awk '{print $1}')"
|
||||||
|
[[ "$app_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||||
|
[[ "$db_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||||
|
|
||||||
|
image_manifest="$fixture/output/releases/incoming/who_need_help-$target_commit-test-images.manifest"
|
||||||
|
printf '%s\n' \
|
||||||
|
'format=1' \
|
||||||
|
'deployment=test' \
|
||||||
|
"commit=$target_commit" \
|
||||||
|
'platform=linux/amd64' \
|
||||||
|
'topology=compact' \
|
||||||
|
'image_count=2' \
|
||||||
|
"image=who-need-help:test-${target_commit:0:12}|$app_image_id" \
|
||||||
|
"image=who-need-help:postgis-test-${target_commit:0:12}|$db_image_id" \
|
||||||
|
>"$image_manifest"
|
||||||
|
|
||||||
|
backup="$fixture/output/backups/test/pre-release.dump"
|
||||||
|
printf 'isolated test release drill backup\n' >"$backup"
|
||||||
|
backup_hash=$(sha256sum "$backup" | awk '{print $1}')
|
||||||
|
printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256"
|
||||||
|
chmod 600 "$bundle" "$bundle.sha256" "$image_archive" \
|
||||||
|
"$image_archive.sha256" "$image_manifest" "$backup" "$backup.sha256"
|
||||||
|
|
||||||
|
for script in validate-test-env.sh verify-realtime-cluster.sh \
|
||||||
|
verify-beam-runtime.sh check-database.sh; do
|
||||||
|
install -m 755 /dev/null "$fixture/scripts/$script"
|
||||||
|
printf '%s\n' '#!/bin/sh' 'exit 0' >"$fixture/scripts/$script"
|
||||||
|
done
|
||||||
|
|
||||||
|
install -m 755 /dev/null "$fixture/scripts/restore-drill-compose.sh"
|
||||||
|
printf '%s\n' \
|
||||||
|
'#!/bin/sh' \
|
||||||
|
'set -eu' \
|
||||||
|
"printf 'restore-drill:%s\\n' \"\$1\" >>\"\$MOCK_COMMAND_LOG\"" \
|
||||||
|
>"$fixture/scripts/restore-drill-compose.sh"
|
||||||
|
|
||||||
|
install -m 755 /dev/null "$fixture/scripts/set-deployment-revision.sh"
|
||||||
|
cat >"$fixture/scripts/set-deployment-revision.sh" <<'EOF'
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
env_file=$1
|
||||||
|
short=${MOCK_TARGET_COMMIT%${MOCK_TARGET_COMMIT#????????????}}
|
||||||
|
sed -i \
|
||||||
|
-e "s|^APP_IMAGE=.*|APP_IMAGE=who-need-help:test-$short|" \
|
||||||
|
-e "s|^SOCKET_PROXY_IMAGE=.*|SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-$short|" \
|
||||||
|
-e "s|^POSTGIS_IMAGE=.*|POSTGIS_IMAGE=who-need-help:postgis-test-$short|" \
|
||||||
|
"$env_file"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
install -m 755 /dev/null "$fixture/scripts/compose.sh"
|
||||||
|
cat >"$fixture/scripts/compose.sh" <<'EOF'
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
env_file=$1
|
||||||
|
shift
|
||||||
|
case "$*" in
|
||||||
|
'config --quiet') exit 0 ;;
|
||||||
|
'ps -q db') printf 'db-1\n'; exit 0 ;;
|
||||||
|
'ps -q app') printf 'app-1\n'; exit 0 ;;
|
||||||
|
'stop app')
|
||||||
|
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
'up -d --no-build --wait db')
|
||||||
|
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
'run --rm --no-deps --interactive=false migrate')
|
||||||
|
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
'up -d --no-deps --no-build --force-recreate --wait app')
|
||||||
|
printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG"
|
||||||
|
if [ "${MOCK_FAIL_APP_UP:-}" = once ] &&
|
||||||
|
[ ! -e "$MOCK_FAIL_APP_MARKER" ]; then
|
||||||
|
: >"$MOCK_FAIL_APP_MARKER"
|
||||||
|
exit 23
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
printf 'Unexpected compose invocation: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
EOF
|
||||||
|
|
||||||
|
printf '%s\n' "$current_commit" >"$fixture/git-state"
|
||||||
|
install -m 755 /dev/null "$mock_bin/git"
|
||||||
|
cat >"$mock_bin/git" <<'EOF'
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
case " $* " in
|
||||||
|
*' status --porcelain --untracked-files=normal '*) exit 0 ;;
|
||||||
|
*' rev-parse --verify HEAD '*) cat "$MOCK_GIT_STATE"; exit 0 ;;
|
||||||
|
*' rev-parse refs/wnh/test-releases/'*'^{commit} '*) printf '%s\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
|
||||||
|
*' bundle verify '*) exit 0 ;;
|
||||||
|
*' bundle list-heads '*) printf '%s HEAD\n' "$MOCK_TARGET_COMMIT"; exit 0 ;;
|
||||||
|
*' fetch '*) printf 'git:%s\n' "$*" >>"$MOCK_COMMAND_LOG"; exit 0 ;;
|
||||||
|
*' merge-base --is-ancestor '*) exit 0 ;;
|
||||||
|
*' show refs/wnh/test-releases/'*':scripts/release-migration-policy.sh '*)
|
||||||
|
cat <<'POLICY'
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
printf 'migration_policy=%s\n' "$MOCK_MIGRATION_POLICY"
|
||||||
|
printf 'migration_versions=20260101000000:%s\n' "$MOCK_MIGRATION_POLICY"
|
||||||
|
printf 'migration_count=1\n'
|
||||||
|
POLICY
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*' checkout --detach refs/wnh/test-releases/'*)
|
||||||
|
printf '%s\n' "$MOCK_TARGET_COMMIT" >"$MOCK_GIT_STATE"
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*" checkout --detach $MOCK_CURRENT_COMMIT "*)
|
||||||
|
printf '%s\n' "$MOCK_CURRENT_COMMIT" >"$MOCK_GIT_STATE"
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
printf 'Unexpected git invocation: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
EOF
|
||||||
|
|
||||||
|
install -m 755 /dev/null "$mock_bin/docker"
|
||||||
|
cat >"$mock_bin/docker" <<'EOF'
|
||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
if [ "$1" = inspect ]; then
|
||||||
|
case "$3" in
|
||||||
|
'{{.State.Status}}') printf 'running\n' ;;
|
||||||
|
'{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;;
|
||||||
|
'{{.Config.Image}}')
|
||||||
|
if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then
|
||||||
|
printf 'who-need-help:test-wrong\n'
|
||||||
|
elif [ "$4" = db-1 ]; then
|
||||||
|
awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0, index($0, "=") + 1)}' \
|
||||||
|
"$MOCK_ENV_FILE"
|
||||||
|
else
|
||||||
|
awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \
|
||||||
|
"$MOCK_ENV_FILE"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
'{{.Image}}')
|
||||||
|
if [ "$4" = db-1 ]; then
|
||||||
|
printf '%s\n' "$DB_IMAGE_ID"
|
||||||
|
else
|
||||||
|
printf '%s\n' "$APP_IMAGE_ID"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = --format ]; then
|
||||||
|
image=$5
|
||||||
|
case "$4" in
|
||||||
|
'{{.Id}}')
|
||||||
|
case "$image" in
|
||||||
|
who-need-help:postgis-test-*) printf '%s\n' "$DB_IMAGE_ID" ;;
|
||||||
|
*) printf '%s\n' "$APP_IMAGE_ID" ;;
|
||||||
|
esac
|
||||||
|
;;
|
||||||
|
'{{.Os}}/{{.Architecture}}') printf 'linux/amd64\n' ;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "$1" = load ]; then
|
||||||
|
cat >/dev/null
|
||||||
|
printf 'docker:load\n' >>"$MOCK_COMMAND_LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
printf 'Unexpected docker invocation: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
EOF
|
||||||
|
|
||||||
|
for command in curl jq pg_restore; do
|
||||||
|
install -m 755 /dev/null "$mock_bin/$command"
|
||||||
|
printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command"
|
||||||
|
done
|
||||||
|
|
||||||
|
touch "$fixture/mock-commands.log"
|
||||||
|
chmod 600 "$fixture/mock-commands.log" "$fixture/git-state"
|
||||||
|
|
||||||
|
container_env=(
|
||||||
|
--env "MOCK_TARGET_COMMIT=$target_commit"
|
||||||
|
--env "MOCK_CURRENT_COMMIT=$current_commit"
|
||||||
|
--env "MOCK_GIT_STATE=$remote_root/git-state"
|
||||||
|
--env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log"
|
||||||
|
--env "MOCK_ENV_FILE=$remote_root/.env"
|
||||||
|
--env "APP_IMAGE_ID=$app_image_id"
|
||||||
|
--env "DB_IMAGE_ID=$db_image_id"
|
||||||
|
--env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||||
|
)
|
||||||
|
container_mounts=(
|
||||||
|
--volume "$fixture:$remote_root"
|
||||||
|
--volume "$mock_bin:/mock-bin:ro"
|
||||||
|
--volume "$ROOT/scripts/test-release-remote.sh:/runner/test-release-remote.sh:ro"
|
||||||
|
)
|
||||||
|
|
||||||
|
run_release() {
|
||||||
|
local policy=$1
|
||||||
|
shift
|
||||||
|
docker run --rm \
|
||||||
|
--network none \
|
||||||
|
--user "$(id -u):$(id -g)" \
|
||||||
|
--read-only \
|
||||||
|
--tmpfs /tmp:rw,nosuid,nodev,noexec \
|
||||||
|
--cap-drop ALL \
|
||||||
|
--security-opt no-new-privileges \
|
||||||
|
--env "MOCK_MIGRATION_POLICY=$policy" \
|
||||||
|
--env "WNH_TEST_RELEASE_CONFIRM=$release_confirmation" \
|
||||||
|
--env "WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation" \
|
||||||
|
"$@" \
|
||||||
|
"${container_env[@]}" \
|
||||||
|
"${container_mounts[@]}" \
|
||||||
|
"$BASE_IMAGE" \
|
||||||
|
bash -c 'bash -s -- "$@" < /runner/test-release-remote.sh' _ \
|
||||||
|
apply "$remote_root" test.whoneedhelp.com \
|
||||||
|
"$remote_root/output/releases/incoming/$(basename -- "$bundle")" \
|
||||||
|
"$target_commit" \
|
||||||
|
"$remote_root/output/backups/test/$(basename -- "$backup")" \
|
||||||
|
"$policy" \
|
||||||
|
"$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \
|
||||||
|
"$remote_root/output/releases/incoming/$(basename -- "$image_manifest")"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_release forward_only >"$run_dir/forward-success.out"
|
||||||
|
grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null
|
||||||
|
grep -Fx "POSTGIS_IMAGE=who-need-help:postgis-test-${target_commit:0:12}" "$fixture/.env" >/dev/null
|
||||||
|
grep -Fx 'docker:load' "$fixture/mock-commands.log" >/dev/null
|
||||||
|
grep -F 'restore-drill:' "$fixture/mock-commands.log" >/dev/null
|
||||||
|
grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null
|
||||||
|
grep -F 'compose:run --rm --no-deps --interactive=false migrate' \
|
||||||
|
"$fixture/mock-commands.log" >/dev/null
|
||||||
|
grep -F 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
|
||||||
|
"$fixture/mock-commands.log" >/dev/null
|
||||||
|
if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then
|
||||||
|
echo "Test release drill unexpectedly built images on the target host." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
grep -R -F 'status=success' \
|
||||||
|
"$fixture/output/releases" --include rollback-manifest.txt >/dev/null
|
||||||
|
|
||||||
|
write_old_env
|
||||||
|
printf '%s\n' "$current_commit" >"$fixture/git-state"
|
||||||
|
: >"$fixture/mock-commands.log"
|
||||||
|
rm -f "$fixture/app-up-failed"
|
||||||
|
set +e
|
||||||
|
run_release forward_only \
|
||||||
|
--env MOCK_FAIL_APP_UP=once \
|
||||||
|
--env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \
|
||||||
|
>"$run_dir/forward-failure.out" 2>&1
|
||||||
|
forward_status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$forward_status" -ne 0 ]] || {
|
||||||
|
echo "Forward-only test drill did not surface the startup failure." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
grep -F 'automatic old-image restart is blocked' "$run_dir/forward-failure.out" >/dev/null
|
||||||
|
grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null
|
||||||
|
test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
|
||||||
|
"$fixture/mock-commands.log")" = 1
|
||||||
|
|
||||||
|
write_old_env
|
||||||
|
printf '%s\n' "$current_commit" >"$fixture/git-state"
|
||||||
|
: >"$fixture/mock-commands.log"
|
||||||
|
rm -f "$fixture/app-up-failed"
|
||||||
|
set +e
|
||||||
|
run_release application_safe \
|
||||||
|
--env MOCK_FAIL_APP_UP=once \
|
||||||
|
--env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \
|
||||||
|
>"$run_dir/safe-failure.out" 2>&1
|
||||||
|
safe_status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$safe_status" -ne 0 ]] || {
|
||||||
|
echo "Application-safe test drill did not surface the startup failure." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
grep -F 'restoring the previous revision' "$run_dir/safe-failure.out" >/dev/null
|
||||||
|
grep -Fx "APP_IMAGE=who-need-help:test-${current_commit:0:12}" "$fixture/.env" >/dev/null
|
||||||
|
grep -Fx "$current_commit" "$fixture/git-state" >/dev/null
|
||||||
|
test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
|
||||||
|
"$fixture/mock-commands.log")" = 2
|
||||||
|
|
||||||
|
echo "Isolated test release success/forward-only/safe-recovery drill passed."
|
||||||
389
scripts/test-release-remote.sh
Executable file
389
scripts/test-release-remote.sh
Executable file
|
|
@ -0,0 +1,389 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
action=${1:-}
|
||||||
|
root=${2:-/srv/who_need_help-test}
|
||||||
|
expected_domain=${3:-test.whoneedhelp.com}
|
||||||
|
bundle=${4:-}
|
||||||
|
target_commit=${5:-}
|
||||||
|
backup=${6:-}
|
||||||
|
expected_migration_policy=${7:-}
|
||||||
|
image_archive=${8:-}
|
||||||
|
image_manifest=${9:-}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: $0 plan /srv/who_need_help-test test.whoneedhelp.com" >&2
|
||||||
|
echo " $0 apply /srv/who_need_help-test test.whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
case "$action" in
|
||||||
|
plan | apply) ;;
|
||||||
|
*) usage; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
for command in curl docker git gzip jq pg_restore realpath sha256sum; do
|
||||||
|
command -v "$command" >/dev/null 2>&1 || {
|
||||||
|
echo "Required test release command is unavailable: $command" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
root=$(realpath --canonicalize-existing "$root")
|
||||||
|
[[ "$root" == /srv/who_need_help-test ]] || {
|
||||||
|
echo "Refusing a test release outside /srv/who_need_help-test." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
env_file="$root/.env"
|
||||||
|
[[ -f "$env_file" && "$(stat -c '%a' "$env_file")" == 600 ]] || {
|
||||||
|
echo "Test .env is missing or does not have mode 0600." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
read_value() {
|
||||||
|
local key=$1
|
||||||
|
awk -v key="$key" '
|
||||||
|
index($0, key "=") == 1 {
|
||||||
|
print substr($0, length(key) + 2)
|
||||||
|
found = 1
|
||||||
|
exit
|
||||||
|
}
|
||||||
|
END { if (!found) exit 1 }
|
||||||
|
' "$env_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
deployment_environment=$(read_value DEPLOYMENT_ENV)
|
||||||
|
compose_project=$(read_value COMPOSE_PROJECT_NAME)
|
||||||
|
database_mode=$(read_value DATABASE_MODE)
|
||||||
|
app_topology=$(read_value APP_TOPOLOGY)
|
||||||
|
phx_host=$(read_value PHX_HOST)
|
||||||
|
public_origin=$(read_value WNH_BASE_URL)
|
||||||
|
current_commit=$(git -C "$root" rev-parse --verify HEAD)
|
||||||
|
|
||||||
|
[[ "$deployment_environment" == test ]] || {
|
||||||
|
echo "DEPLOYMENT_ENV is not test." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$compose_project" == who_need_help_test ]] || {
|
||||||
|
echo "Unexpected test Compose project." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$database_mode" == container ]] || {
|
||||||
|
echo "The verified test workflow expects DATABASE_MODE=container." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$phx_host" == "$expected_domain" &&
|
||||||
|
"$public_origin" == "https://$expected_domain" ]] || {
|
||||||
|
echo "Test origin does not match the expected domain." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ]] || {
|
||||||
|
echo "Test checkout has uncommitted files." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
"$root/scripts/compose.sh" "$env_file" config --quiet
|
||||||
|
|
||||||
|
case "$app_topology" in
|
||||||
|
compact)
|
||||||
|
expected_services=(app)
|
||||||
|
runtime_services=(app)
|
||||||
|
;;
|
||||||
|
split)
|
||||||
|
expected_services=(web worker)
|
||||||
|
runtime_services=(docker-api-proxy proxy web worker)
|
||||||
|
;;
|
||||||
|
*) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
for service in db "${expected_services[@]}"; do
|
||||||
|
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
|
||||||
|
[[ ${#containers[@]} -gt 0 ]] || {
|
||||||
|
echo "Test service is not running: $service" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
for container in "${containers[@]}"; do
|
||||||
|
state=$(docker inspect --format '{{.State.Status}}' "$container")
|
||||||
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
||||||
|
[[ "$state" == running && "$health" == healthy ]] || {
|
||||||
|
echo "Test container is not healthy: $service" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
curl --fail --silent --show-error --max-time 15 \
|
||||||
|
"https://$expected_domain/healthz/ready" >/dev/null
|
||||||
|
|
||||||
|
printf 'Test checkout: %s\n' "$root"
|
||||||
|
printf 'Current commit: %s\n' "$current_commit"
|
||||||
|
printf 'Compose project: %s\n' "$compose_project"
|
||||||
|
printf 'Topology: %s\n' "$app_topology"
|
||||||
|
printf 'Database mode: %s\n' "$database_mode"
|
||||||
|
printf 'Public readiness: passed\n'
|
||||||
|
df -h "$root" /var/lib/docker 2>/dev/null || df -h "$root"
|
||||||
|
|
||||||
|
if [[ "$action" == plan ]]; then
|
||||||
|
echo "Read-only test release scope check passed."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
|
||||||
|
-z "$expected_migration_policy" || -z "$image_archive" ||
|
||||||
|
-z "$image_manifest" ]]; then
|
||||||
|
usage
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
expected_confirmation="$expected_domain:$target_commit"
|
||||||
|
[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$expected_confirmation" ]] || {
|
||||||
|
echo "Set WNH_TEST_RELEASE_CONFIRM=$expected_confirmation for the approved test release." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
|
||||||
|
"$image_archive" "$image_archive.sha256" "$image_manifest"; do
|
||||||
|
[[ -f "$required_file" ]] || {
|
||||||
|
echo "Required test release evidence is missing: $required_file" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$(dirname -- "$bundle")"
|
||||||
|
sha256sum --check "$(basename -- "$bundle.sha256")" >/dev/null
|
||||||
|
)
|
||||||
|
(
|
||||||
|
cd "$(dirname -- "$backup")"
|
||||||
|
sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null
|
||||||
|
)
|
||||||
|
pg_restore --list "$backup" >/dev/null
|
||||||
|
(
|
||||||
|
cd "$(dirname -- "$image_archive")"
|
||||||
|
sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null
|
||||||
|
)
|
||||||
|
gzip -t "$image_archive"
|
||||||
|
grep -Fx 'format=1' "$image_manifest" >/dev/null
|
||||||
|
grep -Fx 'deployment=test' "$image_manifest" >/dev/null
|
||||||
|
grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
|
||||||
|
grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
|
||||||
|
git -C "$root" bundle verify "$bundle" >/dev/null
|
||||||
|
|
||||||
|
bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}')
|
||||||
|
[[ "$bundle_head" == "$target_commit" ]] || {
|
||||||
|
echo "Bundle HEAD does not match the approved test commit." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
release_ref="refs/wnh/test-releases/$target_commit"
|
||||||
|
git -C "$root" fetch "$bundle" "HEAD:$release_ref"
|
||||||
|
[[ "$(git -C "$root" rev-parse "$release_ref^{commit}")" == "$target_commit" ]] || {
|
||||||
|
echo "Fetched test release ref does not match the approved commit." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || {
|
||||||
|
echo "Test updates must be a fast-forward from the deployed commit." >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
policy_script=$(mktemp)
|
||||||
|
trap 'rm -f "$policy_script"' EXIT HUP INT TERM
|
||||||
|
git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script"
|
||||||
|
chmod 700 "$policy_script"
|
||||||
|
migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root")
|
||||||
|
rm -f "$policy_script"
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
printf '%s\n' "$migration_policy_output"
|
||||||
|
migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output")
|
||||||
|
[[ "$migration_policy" == "$expected_migration_policy" ]] || {
|
||||||
|
echo "Remote migration policy does not match the locally approved policy." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "$migration_policy" == forward_only ]]; then
|
||||||
|
forward_confirmation="$expected_domain:$target_commit:forward-only"
|
||||||
|
[[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
|
||||||
|
echo "Set WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation for this test schema boundary." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
|
||||||
|
release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}"
|
||||||
|
release_dir="$root/output/releases/$release_id"
|
||||||
|
mkdir -p "$release_dir"
|
||||||
|
chmod 700 "$root/output" "$root/output/releases" "$release_dir"
|
||||||
|
rollback_manifest="$release_dir/rollback-manifest.txt"
|
||||||
|
{
|
||||||
|
printf 'previous_commit=%s\n' "$current_commit"
|
||||||
|
printf 'target_commit=%s\n' "$target_commit"
|
||||||
|
printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)"
|
||||||
|
printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)"
|
||||||
|
printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)"
|
||||||
|
printf 'migration_policy=%s\n' "$migration_policy"
|
||||||
|
printf 'database_backup=%s\n' "$backup"
|
||||||
|
printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
printf 'status=started\n'
|
||||||
|
} >"$rollback_manifest"
|
||||||
|
chmod 600 "$rollback_manifest"
|
||||||
|
|
||||||
|
revision_changed=false
|
||||||
|
migration_started=false
|
||||||
|
|
||||||
|
restore_previous_revision() {
|
||||||
|
local temporary
|
||||||
|
temporary=$(mktemp "$root/.env.test-release-rollback.XXXXXX")
|
||||||
|
chmod 600 "$temporary"
|
||||||
|
APP_IMAGE_VALUE=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
||||||
|
SOCKET_PROXY_IMAGE_VALUE=$(awk -F= '$1 == "SOCKET_PROXY_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
||||||
|
POSTGIS_IMAGE_VALUE=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") \
|
||||||
|
awk '
|
||||||
|
BEGIN {
|
||||||
|
replacement["APP_IMAGE"] = ENVIRON["APP_IMAGE_VALUE"]
|
||||||
|
replacement["SOCKET_PROXY_IMAGE"] = ENVIRON["SOCKET_PROXY_IMAGE_VALUE"]
|
||||||
|
replacement["POSTGIS_IMAGE"] = ENVIRON["POSTGIS_IMAGE_VALUE"]
|
||||||
|
}
|
||||||
|
{
|
||||||
|
separator = index($0, "=")
|
||||||
|
key = separator > 1 ? substr($0, 1, separator - 1) : ""
|
||||||
|
print (key in replacement) ? key "=" replacement[key] : $0
|
||||||
|
}
|
||||||
|
' "$env_file" >"$temporary"
|
||||||
|
mv "$temporary" "$env_file"
|
||||||
|
chmod 600 "$env_file"
|
||||||
|
git -C "$root" checkout --detach "$current_commit" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
rollback_runtime() {
|
||||||
|
local status=$?
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
if [[ "$status" -ne 0 && "$revision_changed" == true &&
|
||||||
|
"$migration_policy" == forward_only && "$migration_started" == true ]]; then
|
||||||
|
{
|
||||||
|
printf 'status=forward-only-release-failed\n'
|
||||||
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
printf 'automatic_application_rollback=blocked\n'
|
||||||
|
} >>"$rollback_manifest"
|
||||||
|
echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2
|
||||||
|
elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then
|
||||||
|
echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2
|
||||||
|
restore_previous_revision
|
||||||
|
"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db || true
|
||||||
|
"$root/scripts/compose.sh" "$env_file" \
|
||||||
|
up -d --no-deps --no-build --force-recreate --wait \
|
||||||
|
"${runtime_services[@]}" || true
|
||||||
|
{
|
||||||
|
printf 'status=runtime-rolled-back\n'
|
||||||
|
printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
} >>"$rollback_manifest"
|
||||||
|
fi
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
trap rollback_runtime EXIT HUP INT TERM
|
||||||
|
|
||||||
|
gzip -dc "$image_archive" | docker load >/dev/null
|
||||||
|
git -C "$root" checkout --detach "$release_ref" >/dev/null
|
||||||
|
"$root/scripts/set-deployment-revision.sh" "$env_file"
|
||||||
|
revision_changed=true
|
||||||
|
"$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain"
|
||||||
|
|
||||||
|
expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)")
|
||||||
|
if [[ "$app_topology" == split ]]; then
|
||||||
|
expected_images+=(
|
||||||
|
"$(read_value SOCKET_PROXY_IMAGE)"
|
||||||
|
"$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null
|
||||||
|
test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}"
|
||||||
|
for image in "${expected_images[@]}"; do
|
||||||
|
expected_id=$(awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' "$image_manifest")
|
||||||
|
[[ -n "$expected_id" ]] || {
|
||||||
|
echo "Image manifest is missing the expected image: $image" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || {
|
||||||
|
echo "Loaded test image ID does not match the manifest: $image" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || {
|
||||||
|
echo "Loaded test image is not linux/amd64: $image" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
"$root/scripts/restore-drill-compose.sh" "$backup"
|
||||||
|
|
||||||
|
if [[ "$migration_policy" == forward_only ]]; then
|
||||||
|
echo "Stopping the old test application before the forward-only migration boundary."
|
||||||
|
"$root/scripts/compose.sh" "$env_file" stop "${expected_services[@]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
"$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db
|
||||||
|
|
||||||
|
verify_service_image() {
|
||||||
|
local service=$1 expected_image=$2 require_health=$3
|
||||||
|
local expected_image_id container state health configured_image running_image_id
|
||||||
|
|
||||||
|
expected_image_id=$(docker image inspect --format '{{.Id}}' "$expected_image")
|
||||||
|
mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service")
|
||||||
|
[[ ${#containers[@]} -gt 0 ]] || {
|
||||||
|
echo "Candidate test service has no container: $service" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
for container in "${containers[@]}"; do
|
||||||
|
state=$(docker inspect --format '{{.State.Status}}' "$container")
|
||||||
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
||||||
|
configured_image=$(docker inspect --format '{{.Config.Image}}' "$container")
|
||||||
|
running_image_id=$(docker inspect --format '{{.Image}}' "$container")
|
||||||
|
|
||||||
|
[[ "$state" == running ]] || {
|
||||||
|
echo "Candidate test container is not running: $service" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if [[ "$require_health" == true && "$health" != healthy ]]; then
|
||||||
|
echo "Candidate test container is not healthy: $service" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
[[ "$configured_image" == "$expected_image" &&
|
||||||
|
"$running_image_id" == "$expected_image_id" ]] || {
|
||||||
|
echo "Candidate test service does not use its approved image: $service" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_service_image db "$(read_value POSTGIS_IMAGE)" true
|
||||||
|
migration_started=true
|
||||||
|
"$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate
|
||||||
|
"$root/scripts/check-database.sh" "$env_file" </dev/null
|
||||||
|
"$root/scripts/compose.sh" "$env_file" \
|
||||||
|
up -d --no-deps --no-build --force-recreate --wait "${runtime_services[@]}"
|
||||||
|
|
||||||
|
expected_app_image=$(read_value APP_IMAGE)
|
||||||
|
for service in "${expected_services[@]}"; do
|
||||||
|
verify_service_image "$service" "$expected_app_image" true
|
||||||
|
done
|
||||||
|
if [[ "$app_topology" == split ]]; then
|
||||||
|
verify_service_image docker-api-proxy "$(read_value SOCKET_PROXY_IMAGE)" false
|
||||||
|
verify_service_image proxy \
|
||||||
|
"$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" \
|
||||||
|
false
|
||||||
|
fi
|
||||||
|
|
||||||
|
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-realtime-cluster.sh" compose
|
||||||
|
COMPOSE_PROJECT_NAME="$compose_project" "$root/scripts/verify-beam-runtime.sh" compose
|
||||||
|
curl --fail --silent --show-error --max-time 30 \
|
||||||
|
"https://$expected_domain/healthz/ready" >/dev/null
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'status=success\n'
|
||||||
|
printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
} >>"$rollback_manifest"
|
||||||
|
revision_changed=false
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
|
||||||
|
printf 'Test release completed: %s\n' "$target_commit"
|
||||||
|
printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
|
||||||
|
printf 'Database backup: %s\n' "$backup"
|
||||||
192
scripts/test-release.sh
Executable file
192
scripts/test-release.sh
Executable file
|
|
@ -0,0 +1,192 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
action=${1:-plan}
|
||||||
|
ssh_target=${2:-whoneedhelp}
|
||||||
|
remote_root=${WNH_TEST_REMOTE_ROOT:-/srv/who_need_help-test}
|
||||||
|
production_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production}
|
||||||
|
expected_domain=${WNH_TEST_DOMAIN:-test.whoneedhelp.com}
|
||||||
|
|
||||||
|
case "$action" in
|
||||||
|
plan | prepare | apply) ;;
|
||||||
|
*)
|
||||||
|
echo "Usage: $0 [plan|prepare|apply] [SSH_TARGET]" >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
for command in git gzip mktemp pg_restore realpath scp sha256sum ssh; do
|
||||||
|
command -v "$command" >/dev/null 2>&1 || {
|
||||||
|
echo "Required command is unavailable: $command" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
local_commit=$(git -C "$ROOT" rev-parse --verify HEAD)
|
||||||
|
remote_commit=$(
|
||||||
|
ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"git -C '$remote_root' rev-parse --verify HEAD"
|
||||||
|
)
|
||||||
|
|
||||||
|
printf 'Local candidate commit: %s\n' "$local_commit"
|
||||||
|
printf 'Current test commit: %s\n' "$remote_commit"
|
||||||
|
|
||||||
|
git -C "$ROOT" cat-file -e "$remote_commit^{commit}" 2>/dev/null || {
|
||||||
|
echo "The test commit is not present in the local object database." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
git -C "$ROOT" merge-base --is-ancestor "$remote_commit" "$local_commit" || {
|
||||||
|
echo "The local candidate is not a fast-forward from the test commit." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
printf 'Pending commits: %s\n' \
|
||||||
|
"$(git -C "$ROOT" rev-list --count "$remote_commit..$local_commit")"
|
||||||
|
|
||||||
|
legacy_edge_paths=(compose.edge.yaml deploy/caddy/Caddyfile)
|
||||||
|
if ! git -C "$ROOT" diff --quiet \
|
||||||
|
"$remote_commit" "$local_commit" -- "${legacy_edge_paths[@]}"; then
|
||||||
|
echo "The test application release contains shared edge routing changes." >&2
|
||||||
|
echo "Move route changes through the independent server-edge workflow." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
echo "Shared edge routing files are unchanged; the test release will not manage Caddy."
|
||||||
|
|
||||||
|
migration_policy_output=$(
|
||||||
|
"$ROOT/scripts/release-migration-policy.sh" "$remote_commit" "$local_commit"
|
||||||
|
)
|
||||||
|
printf '%s\n' "$migration_policy_output"
|
||||||
|
migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output")
|
||||||
|
|
||||||
|
plan_failed=0
|
||||||
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"cd '$remote_root' && ./scripts/validate-test-env.sh .env '$expected_domain'"; then
|
||||||
|
plan_failed=1
|
||||||
|
fi
|
||||||
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"cd '$remote_root' && ./scripts/validate-deployment-isolation.sh '$remote_root' '$production_root'"; then
|
||||||
|
plan_failed=1
|
||||||
|
fi
|
||||||
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"bash -s -- plan '$remote_root' '$expected_domain'" \
|
||||||
|
<"$ROOT/scripts/test-release-remote.sh"; then
|
||||||
|
plan_failed=1
|
||||||
|
fi
|
||||||
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"cd '$remote_root' && ./scripts/check-environment-readiness.sh .env --require-server-release"; then
|
||||||
|
plan_failed=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$plan_failed" -ne 0 ]]; then
|
||||||
|
echo "Test release plan has blocking checks; no remote state was changed." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$action" == plan ]]; then
|
||||||
|
echo "Test release plan passed; no remote state was changed."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ -z "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]] || {
|
||||||
|
echo "Refusing to release a dirty checkout." >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact_root=${WNH_TEST_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/test-releases"}
|
||||||
|
case "$artifact_root" in
|
||||||
|
/*) ;;
|
||||||
|
*)
|
||||||
|
echo "WNH_TEST_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
mkdir -p "$artifact_root"
|
||||||
|
artifact_root=$(realpath --canonicalize-existing "$artifact_root")
|
||||||
|
|
||||||
|
WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT="$artifact_root" \
|
||||||
|
"$ROOT/scripts/prepare-production-release.sh"
|
||||||
|
release_dir="$artifact_root/$local_commit"
|
||||||
|
bundle="$release_dir/who_need_help-$local_commit.bundle"
|
||||||
|
image_archive="$release_dir/who_need_help-$local_commit-test-images-linux-amd64.tar.gz"
|
||||||
|
image_checksum="$image_archive.sha256"
|
||||||
|
image_manifest="$release_dir/who_need_help-$local_commit-test-images.manifest"
|
||||||
|
|
||||||
|
test_env=$(mktemp)
|
||||||
|
cleanup_test_env() {
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
rm -f "$test_env"
|
||||||
|
}
|
||||||
|
trap cleanup_test_env EXIT HUP INT TERM
|
||||||
|
scp -p "$ssh_target:$remote_root/.env" "$test_env"
|
||||||
|
chmod 600 "$test_env"
|
||||||
|
WNH_TEST_RELEASE_ARTIFACT_ROOT="$artifact_root" \
|
||||||
|
"$ROOT/scripts/prepare-test-images.sh" "$test_env"
|
||||||
|
cleanup_test_env
|
||||||
|
|
||||||
|
if [[ "$action" == prepare ]]; then
|
||||||
|
echo "Test release artifacts are prepared and verified locally; no remote state was changed."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
remote_main=$(git -C "$ROOT" ls-remote origin refs/heads/main | awk '{print $1}')
|
||||||
|
[[ "$remote_main" == "$local_commit" ]] || {
|
||||||
|
echo "origin/main does not contain the exact approved test candidate." >&2
|
||||||
|
echo "Expected: $local_commit" >&2
|
||||||
|
echo "Observed: ${remote_main:-missing}" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
confirmation="$expected_domain:$local_commit"
|
||||||
|
[[ "${WNH_TEST_RELEASE_CONFIRM:-}" == "$confirmation" ]] || {
|
||||||
|
echo "Test release execution requires exact approval:" >&2
|
||||||
|
echo "WNH_TEST_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
if [[ "$migration_policy" == forward_only ]]; then
|
||||||
|
forward_confirmation="$expected_domain:$local_commit:forward-only"
|
||||||
|
[[ "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}" == "$forward_confirmation" ]] || {
|
||||||
|
echo "This test release contains forward-only migrations." >&2
|
||||||
|
echo "WNH_TEST_FORWARD_ONLY_CONFIRM=$forward_confirmation \\" >&2
|
||||||
|
echo " WNH_TEST_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
|
||||||
|
remote_release_dir="$remote_root/output/releases/incoming"
|
||||||
|
remote_bundle="$remote_release_dir/$(basename -- "$bundle")"
|
||||||
|
remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")"
|
||||||
|
remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")"
|
||||||
|
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||||
|
remote_backup="$remote_root/output/backups/test/pre-$timestamp-${local_commit:0:12}.dump"
|
||||||
|
|
||||||
|
ssh -o BatchMode=yes "$ssh_target" "install -d -m 700 '$remote_release_dir'"
|
||||||
|
scp -p \
|
||||||
|
"$bundle" "$bundle.sha256" \
|
||||||
|
"$image_archive" "$image_checksum" "$image_manifest" \
|
||||||
|
"$ssh_target:$remote_release_dir/"
|
||||||
|
|
||||||
|
ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"cd '$remote_root' && ./scripts/backup-compose.sh '$remote_backup'"
|
||||||
|
|
||||||
|
local_backup_dir="$ROOT/output/test-backups/$timestamp-${local_commit:0:12}"
|
||||||
|
mkdir -p "$local_backup_dir"
|
||||||
|
chmod 700 "$ROOT/output" "$ROOT/output/test-backups" "$local_backup_dir"
|
||||||
|
scp -p \
|
||||||
|
"$ssh_target:$remote_backup" \
|
||||||
|
"$ssh_target:$remote_backup.sha256" \
|
||||||
|
"$local_backup_dir/"
|
||||||
|
(
|
||||||
|
cd "$local_backup_dir"
|
||||||
|
sha256sum --check "$(basename -- "$remote_backup.sha256")" >/dev/null
|
||||||
|
)
|
||||||
|
pg_restore --list "$local_backup_dir/$(basename -- "$remote_backup")" >/dev/null
|
||||||
|
echo "Copied and independently verified the pre-release test backup outside the server."
|
||||||
|
|
||||||
|
quoted_confirmation=$(printf '%q' "$confirmation")
|
||||||
|
quoted_forward_confirmation=$(printf '%q' "${WNH_TEST_FORWARD_ONLY_CONFIRM:-}")
|
||||||
|
ssh -o BatchMode=yes "$ssh_target" \
|
||||||
|
"WNH_TEST_RELEASE_CONFIRM=$quoted_confirmation WNH_TEST_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \
|
||||||
|
<"$ROOT/scripts/test-release-remote.sh"
|
||||||
|
|
||||||
|
echo "Test release and public health verification completed."
|
||||||
|
|
@ -91,6 +91,21 @@ class ProductionReleaseArtifactRootTest(unittest.TestCase):
|
||||||
self.assertEqual(result.returncode, 2)
|
self.assertEqual(result.returncode, 2)
|
||||||
self.assertIn("must be an absolute path", result.stderr)
|
self.assertIn("must be an absolute path", result.stderr)
|
||||||
|
|
||||||
|
def test_untracked_build_input_is_rejected(self):
|
||||||
|
(self.project / "untracked-build-input.txt").write_text(
|
||||||
|
"must not enter an immutable release\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.run_command(
|
||||||
|
[str(self.scripts / "prepare-production-release.sh")],
|
||||||
|
cwd=self.project,
|
||||||
|
env=self.artifact_env(),
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertIn("dirty checkout", result.stderr)
|
||||||
|
|
||||||
def test_bundle_manifest_for_another_commit_is_rejected(self):
|
def test_bundle_manifest_for_another_commit_is_rejected(self):
|
||||||
self.run_command(
|
self.run_command(
|
||||||
[str(self.scripts / "prepare-production-release.sh")],
|
[str(self.scripts / "prepare-production-release.sh")],
|
||||||
|
|
|
||||||
178
test/scripts/test_release_artifact_root_test.py
Normal file
178
test/scripts/test_release_artifact_root_test.py
Normal file
|
|
@ -0,0 +1,178 @@
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import textwrap
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
class TestReleaseArtifactRootTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tempdir = tempfile.TemporaryDirectory()
|
||||||
|
self.base = Path(self.tempdir.name)
|
||||||
|
self.project = self.base / "project"
|
||||||
|
self.scripts = self.project / "scripts"
|
||||||
|
self.scripts.mkdir(parents=True)
|
||||||
|
script = self.scripts / "prepare-test-images.sh"
|
||||||
|
shutil.copy2(ROOT / "scripts" / script.name, script)
|
||||||
|
script.chmod(0o755)
|
||||||
|
|
||||||
|
self.run_command(["git", "init", "--quiet"], cwd=self.project)
|
||||||
|
self.run_command(
|
||||||
|
["git", "config", "user.email", "test-release@example.invalid"],
|
||||||
|
cwd=self.project,
|
||||||
|
)
|
||||||
|
self.run_command(
|
||||||
|
["git", "config", "user.name", "Test release"], cwd=self.project
|
||||||
|
)
|
||||||
|
self.run_command(["git", "add", "scripts"], cwd=self.project)
|
||||||
|
self.run_command(
|
||||||
|
["git", "commit", "--quiet", "-m", "test fixture"], cwd=self.project
|
||||||
|
)
|
||||||
|
self.commit = self.run_command(
|
||||||
|
["git", "rev-parse", "HEAD"], cwd=self.project
|
||||||
|
).stdout.strip()
|
||||||
|
self.artifact_root = self.base / "test-artifacts"
|
||||||
|
|
||||||
|
self.fake_bin = self.base / "bin"
|
||||||
|
self.fake_bin.mkdir()
|
||||||
|
for name in ("docker", "jq"):
|
||||||
|
command = self.fake_bin / name
|
||||||
|
command.write_text("#!/bin/sh\nexit 97\n", encoding="utf-8")
|
||||||
|
command.chmod(0o755)
|
||||||
|
|
||||||
|
self.test_env = self.base / "test.env"
|
||||||
|
self.test_env.write_text(
|
||||||
|
textwrap.dedent(
|
||||||
|
"""\
|
||||||
|
DEPLOYMENT_ENV=test
|
||||||
|
DATABASE_MODE=container
|
||||||
|
APP_TOPOLOGY=compact
|
||||||
|
APP_IMAGE=replace-me
|
||||||
|
SOCKET_PROXY_IMAGE=replace-me
|
||||||
|
POSTGIS_IMAGE=replace-me
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
self.test_env.chmod(0o600)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tempdir.cleanup()
|
||||||
|
|
||||||
|
def run_command(self, command, *, cwd=None, env=None, check=True):
|
||||||
|
return subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=cwd,
|
||||||
|
env=env,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=check,
|
||||||
|
)
|
||||||
|
|
||||||
|
def environment(self):
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["WNH_TEST_RELEASE_ARTIFACT_ROOT"] = str(self.artifact_root)
|
||||||
|
env["PATH"] = f"{self.fake_bin}:{env['PATH']}"
|
||||||
|
return env
|
||||||
|
|
||||||
|
def write_existing_artifact(self):
|
||||||
|
release_dir = self.artifact_root / self.commit
|
||||||
|
release_dir.mkdir(parents=True)
|
||||||
|
archive = (
|
||||||
|
release_dir
|
||||||
|
/ f"who_need_help-{self.commit}-test-images-linux-amd64.tar.gz"
|
||||||
|
)
|
||||||
|
with archive.open("wb") as output:
|
||||||
|
with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed:
|
||||||
|
compressed.write(b"verified test image archive fixture")
|
||||||
|
archive.chmod(0o600)
|
||||||
|
digest = hashlib.sha256(archive.read_bytes()).hexdigest()
|
||||||
|
checksum = Path(f"{archive}.sha256")
|
||||||
|
checksum.write_text(f"{digest} {archive.name}\n", encoding="utf-8")
|
||||||
|
checksum.chmod(0o600)
|
||||||
|
|
||||||
|
short_commit = self.commit[:12]
|
||||||
|
manifest = release_dir / f"who_need_help-{self.commit}-test-images.manifest"
|
||||||
|
manifest.write_text(
|
||||||
|
textwrap.dedent(
|
||||||
|
f"""\
|
||||||
|
format=1
|
||||||
|
deployment=test
|
||||||
|
commit={self.commit}
|
||||||
|
platform=linux/amd64
|
||||||
|
topology=compact
|
||||||
|
image_count=2
|
||||||
|
image=who-need-help:test-{short_commit}|sha256:{'a' * 64}
|
||||||
|
image=who-need-help:postgis-test-{short_commit}|sha256:{'b' * 64}
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
manifest.chmod(0o600)
|
||||||
|
return archive, manifest
|
||||||
|
|
||||||
|
def test_existing_test_archive_is_verified_without_building(self):
|
||||||
|
archive, manifest = self.write_existing_artifact()
|
||||||
|
|
||||||
|
result = self.run_command(
|
||||||
|
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
|
||||||
|
cwd=self.project,
|
||||||
|
env=self.environment(),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn("verifying all metadata", result.stdout)
|
||||||
|
self.assertIn(str(archive), result.stdout)
|
||||||
|
|
||||||
|
manifest.write_text(
|
||||||
|
manifest.read_text(encoding="utf-8").replace(
|
||||||
|
"deployment=test", "deployment=production"
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
rejected = self.run_command(
|
||||||
|
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
|
||||||
|
cwd=self.project,
|
||||||
|
env=self.environment(),
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(rejected.returncode, 2)
|
||||||
|
self.assertIn("wrong deployment identity", rejected.stderr)
|
||||||
|
|
||||||
|
def test_relative_artifact_root_is_rejected(self):
|
||||||
|
env = self.environment()
|
||||||
|
env["WNH_TEST_RELEASE_ARTIFACT_ROOT"] = "relative/test-releases"
|
||||||
|
result = self.run_command(
|
||||||
|
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
|
||||||
|
cwd=self.project,
|
||||||
|
env=env,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
self.assertIn("must be an absolute path", result.stderr)
|
||||||
|
|
||||||
|
def test_untracked_build_input_is_rejected(self):
|
||||||
|
(self.project / "untracked-build-input.txt").write_text(
|
||||||
|
"must not enter an immutable release\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.run_command(
|
||||||
|
[str(self.scripts / "prepare-test-images.sh"), str(self.test_env)],
|
||||||
|
cwd=self.project,
|
||||||
|
env=self.environment(),
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
self.assertIn("dirty checkout", result.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in New Issue
Block a user