From abc1bed14045f4858d4ac8ab9aed26513f0e47ca Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Fri, 28 Aug 2026 00:23:04 +0300 Subject: [PATCH] Harden isolated test and production releases --- README.md | 9 +- docs/operations.md | 50 +++-- scripts/prepare-production-images.sh | 64 +++--- scripts/prepare-production-release.sh | 12 +- scripts/prepare-test-images.sh | 62 +++--- scripts/production-release-drill.sh | 82 ++++++- scripts/production-release-remote.sh | 196 ++++++++-------- scripts/production-release.sh | 17 +- scripts/test-release-drill.sh | 72 +++++- scripts/test-release-remote.sh | 210 +++++++++++------- scripts/test-release.sh | 20 +- .../production_release_artifact_root_test.py | 16 ++ .../test_release_artifact_root_test.py | 16 ++ 13 files changed, 558 insertions(+), 268 deletions(-) diff --git a/README.md b/README.md index 0510bf1..850daa0 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,12 @@ policies are deliberately not claimed as complete. ## Fast start with Docker Compose The public single-server path uses the compact application topology plus an -independent server-level Caddy edge. Production and test can run as isolated -Compose projects with distinct PostGIS volumes and secrets while sharing only a -Docker network used for HTTPS reverse proxying. See the +independent server-level Caddy edge. Production and test run as isolated +Compose projects with distinct checkouts, configuration, images, databases, +volumes, OAuth clients, and email credentials while sharing only a Docker +network used for HTTPS reverse proxying. A candidate is committed and pushed, +released and verified on the public test site, and only that exact verified +commit SHA is then eligible for production promotion. See the [operations runbook](docs/operations.md#two-independent-checkouts-and-one-env-in-each) for the verified order of operations. Redis is not a project dependency. diff --git a/docs/operations.md b/docs/operations.md index 56d7c4f..01de3a2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -110,10 +110,11 @@ Those files are generated automatically, never copied to a server, and do not represent dev, test, or production. The one ignored `.env` at each checkout root remains the only application/deployment configuration. -The current submitted deployment still has a legacy shared Caddy container -created from the production checkout. Keep it running and unchanged while the -judging test URL is frozen. Application release and rollback scripts do not -build, recreate, or select an image for that container. +The server still has a legacy shared Caddy container created from the +production checkout. Application release and rollback scripts do not build, +recreate, or select an image for that container. Test and production releases +change only their own application checkout, images, Compose project, database, +and release evidence. The replacement is a separate server-level `server_edge` project with its own single mode-`0600` `.env`, route directory, Caddy image, certificate volumes, @@ -380,9 +381,21 @@ WNH_TEST_FORWARD_ONLY_CONFIRM="test.whoneedhelp.com:$candidate:forward-only" \ ``` Omit `WNH_TEST_FORWARD_ONLY_CONFIRM` when the read-only plan reports -`migration_policy=application_safe`. The workflow refuses shared Caddy changes, -requires a fast-forward from the deployed test commit, preserves the single -test `.env`, and never addresses the production Compose project or database. +`migration_policy=application_safe`. This verified single-server workflow +requires `APP_TOPOLOGY=compact`; split Compose and Kubernetes remain separate +deployment paths. The workflow refuses shared Caddy changes, requires a +fast-forward from the deployed test commit, preserves the single test `.env`, +and never addresses the production Compose project or database. A nonblocking +lock at `.git/wnh-test-release.lock` rejects overlapping test releases before +they mutate the checkout or runtime. + +For an `application_safe` failure after the candidate checkout is selected, +automatic recovery restores the exact prior commit, immutable application and +PostGIS tags, and their recorded image IDs. Recovery is successful only after +the old containers and public readiness are verified. A recovery failure is +recorded as `status=rollback-failed` and the release exits unsuccessfully; it is +never reported as a successful rollback. A `forward_only` failure after +migration begins does not start the old application against the new schema. Do not use `deploy-up.sh` for an ordinary public test update on the small server: that command intentionally includes `--build` and therefore builds the release on the target host. @@ -1356,11 +1369,13 @@ interrupt, refuses cross-fixture relationships, deletes only matching jobs and records, and verifies that the run prefix is absent. It never resets the database. Evidence is stored below `output/production-full-e2e//`. -## Production release without pushing the frozen repository +## Promote the revision verified in public test to production -The post-submission workflow keeps the public Git repository and -`test.whoneedhelp.com` untouched. A clean local commit is packaged as a -verified Git bundle and transferred directly over SSH to only +Push a clean candidate commit, release it to `test.whoneedhelp.com`, and finish +the browser, Android, database, email, and operational checks there first. +Production must receive that exact full commit SHA, not a rebuilt or amended +variant. The production release packages the selected clean commit as a +verified Git bundle and transfers it directly over SSH to only `/srv/who_need_help-production`: ```bash @@ -1380,6 +1395,11 @@ allows only the absent Play App Signing certificate; the stricter publishing Android through Google Play. The plan neither uploads a bundle nor creates a backup. +The verified single-server production workflow requires +`APP_TOPOLOGY=compact`. A nonblocking lock at +`.git/wnh-production-release.lock` rejects overlapping production releases +before they mutate the checkout or runtime. + Prepare and verify the immutable Git bundle and `linux/amd64` image archive on the development workstation before authorising any production mutation: @@ -1464,9 +1484,11 @@ Restarting an older image against a potentially incompatible schema is never automatic. For an `application_safe` release, an application startup failure restores the -previous immutable application image tags and attempts to recover public -readiness through the unchanged edge. A `forward_only` release never starts the old application -after migration begins. Neither path reverses Git source or Ecto migrations +previous commit and immutable application image tag, verifies its recorded +image ID, and verifies public readiness through the unchanged edge. If any +recovery check fails, the manifest records `status=rollback-failed` and the +release remains failed. A `forward_only` release never starts the old +application after migration begins. Neither path reverses Ecto migrations automatically. The per-release rollback manifest and backup paths are recorded below the production checkout's ignored `output/releases/`. The copied backup is separate from the production host, but a long-term encrypted off-site diff --git a/scripts/prepare-production-images.sh b/scripts/prepare-production-images.sh index b7997be..ea55b5a 100755 --- a/scripts/prepare-production-images.sh +++ b/scripts/prepare-production-images.sh @@ -10,7 +10,7 @@ if [[ -z "$source_env" || ! -f "$source_env" ]]; then exit 2 fi -for command in docker gzip jq sha256sum; do +for command in docker gzip sha256sum; do command -v "$command" >/dev/null 2>&1 || { echo "Required command is unavailable: $command" >&2 exit 2 @@ -39,26 +39,43 @@ archive="$release_dir/who_need_help-$commit-images-linux-amd64.tar.gz" checksum="$archive.sha256" manifest="$release_dir/who_need_help-$commit-images.manifest" -mkdir -p "$release_dir" -chmod 700 "$artifact_root" "$release_dir" +if [[ -e "$release_dir" ]]; then + [[ ! -L "$release_dir" && -d "$release_dir" && + "$(stat -c '%u' "$release_dir")" == "$(id -u)" && + "$(stat -c '%a' "$release_dir")" == 700 ]] || { + echo "Existing production release directory must be an owned mode-0700 directory: $release_dir" >&2 + exit 2 + } +else + install -d -m 700 "$release_dir" +fi build_env=$(mktemp "$release_dir/.production-image-build.XXXXXX") +manifest_tmp= +archive_tmp= cleanup() { + local status=$? trap - EXIT HUP INT TERM - rm -f "$build_env" + rm -f "$build_env" "${manifest_tmp:-}" "${archive_tmp:-}" + exit "$status" } -trap cleanup EXIT HUP INT TERM +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM install -m 600 "$source_env" "$build_env" read_value() { local key=$1 awk -v key="$key" ' index($0, key "=") == 1 { - print substr($0, length(key) + 2) - found = 1 - exit + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1) exit 1 + print value } - END { if (!found) exit 1 } ' "$build_env" } @@ -92,29 +109,14 @@ replace_value SOCKET_PROXY_IMAGE \ replace_value POSTGIS_IMAGE "who-need-help:postgis-production-$short_commit" topology=$(read_value APP_TOPOLOGY) -case "$topology" in - compact) - build_services=(migrate) - ;; - split) - build_services=(docker-api-proxy proxy migrate) - ;; - *) - echo "APP_TOPOLOGY must be compact or split." >&2 - exit 2 - ;; -esac +[[ "$topology" == compact ]] || { + echo "The verified single-server production release workflow requires APP_TOPOLOGY=compact." >&2 + exit 2 +} +build_services=(migrate) app_image=$(read_value APP_IMAGE) images=("$app_image") -if [[ "$topology" == split ]]; then - socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE) - proxy_image=$( - "$ROOT/scripts/compose.sh" "$build_env" config --format json | - jq -er '.services.proxy.image' - ) - images+=("$socket_proxy_image" "$proxy_image") -fi if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then [[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || { @@ -131,7 +133,6 @@ else manifest_tmp=$(mktemp "$release_dir/.production-images-manifest.XXXXXX") archive_tmp=$(mktemp "$release_dir/.production-images-archive.XXXXXX") - trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM { printf 'format=1\n' @@ -211,7 +212,8 @@ for image in "${images[@]}"; do ' "$manifest" done -cleanup +rm -f "$build_env" "${manifest_tmp:-}" "${archive_tmp:-}" +trap - EXIT HUP INT TERM printf 'Production image archive: %s\n' "$archive" printf 'Image archive checksum: %s\n' "$checksum" printf 'Image manifest: %s\n' "$manifest" diff --git a/scripts/prepare-production-release.sh b/scripts/prepare-production-release.sh index 81973ea..7086a49 100755 --- a/scripts/prepare-production-release.sh +++ b/scripts/prepare-production-release.sh @@ -26,8 +26,16 @@ bundle="$release_dir/who_need_help-$commit.bundle" checksum="$bundle.sha256" manifest="$release_dir/manifest.txt" -mkdir -p "$release_dir" -chmod 700 "$artifact_root" "$release_dir" +if [[ -e "$release_dir" ]]; then + [[ ! -L "$release_dir" && -d "$release_dir" && + "$(stat -c '%u' "$release_dir")" == "$(id -u)" && + "$(stat -c '%a' "$release_dir")" == 700 ]] || { + echo "Existing release directory must be an owned mode-0700 directory: $release_dir" >&2 + exit 2 + } +else + install -d -m 700 "$release_dir" +fi if [[ -e "$bundle" || -e "$checksum" || -e "$manifest" ]]; then echo "Release package already exists; verifying it instead of overwriting it." diff --git a/scripts/prepare-test-images.sh b/scripts/prepare-test-images.sh index abb948d..cf19df2 100755 --- a/scripts/prepare-test-images.sh +++ b/scripts/prepare-test-images.sh @@ -39,26 +39,43 @@ archive="$release_dir/who_need_help-$commit-test-images-linux-amd64.tar.gz" checksum="$archive.sha256" manifest="$release_dir/who_need_help-$commit-test-images.manifest" -mkdir -p "$release_dir" -chmod 700 "$artifact_root" "$release_dir" +if [[ -e "$release_dir" ]]; then + [[ ! -L "$release_dir" && -d "$release_dir" && + "$(stat -c '%u' "$release_dir")" == "$(id -u)" && + "$(stat -c '%a' "$release_dir")" == 700 ]] || { + echo "Existing test release directory must be an owned mode-0700 directory: $release_dir" >&2 + exit 2 + } +else + install -d -m 700 "$release_dir" +fi build_env=$(mktemp "$release_dir/.test-image-build.XXXXXX") +manifest_tmp= +archive_tmp= cleanup() { + local status=$? trap - EXIT HUP INT TERM - rm -f "$build_env" + rm -f "$build_env" "${manifest_tmp:-}" "${archive_tmp:-}" + exit "$status" } -trap cleanup EXIT HUP INT TERM +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM install -m 600 "$source_env" "$build_env" read_value() { local key=$1 awk -v key="$key" ' index($0, key "=") == 1 { - print substr($0, length(key) + 2) - found = 1 - exit + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1) exit 1 + print value } - END { if (!found) exit 1 } ' "$build_env" } @@ -90,30 +107,15 @@ replace_value SOCKET_PROXY_IMAGE \ replace_value POSTGIS_IMAGE "who-need-help:postgis-test-$short_commit" topology=$(read_value APP_TOPOLOGY) -case "$topology" in - compact) - build_services=(migrate db) - ;; - split) - build_services=(docker-api-proxy proxy migrate db) - ;; - *) - echo "APP_TOPOLOGY must be compact or split." >&2 - exit 2 - ;; -esac +[[ "$topology" == compact ]] || { + echo "The verified single-server test release workflow requires APP_TOPOLOGY=compact." >&2 + exit 2 +} +build_services=(migrate db) app_image=$(read_value APP_IMAGE) postgis_image=$(read_value POSTGIS_IMAGE) images=("$app_image" "$postgis_image") -if [[ "$topology" == split ]]; then - socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE) - proxy_image=$( - "$ROOT/scripts/compose.sh" "$build_env" config --format json | - jq -er '.services.proxy.image' - ) - images+=("$socket_proxy_image" "$proxy_image") -fi if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then [[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || { @@ -130,7 +132,6 @@ else manifest_tmp=$(mktemp "$release_dir/.test-images-manifest.XXXXXX") archive_tmp=$(mktemp "$release_dir/.test-images-archive.XXXXXX") - trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM { printf 'format=1\n' @@ -215,7 +216,8 @@ expected_checksum="$archive_hash $(basename -- "$archive")" exit 2 } -cleanup +rm -f "$build_env" "${manifest_tmp:-}" "${archive_tmp:-}" +trap - EXIT HUP INT TERM printf 'Test image archive: %s\n' "$archive" printf 'Image archive checksum: %s\n' "$checksum" printf 'Image manifest: %s\n' "$manifest" diff --git a/scripts/production-release-drill.sh b/scripts/production-release-drill.sh index 6b8fc29..c70da46 100755 --- a/scripts/production-release-drill.sh +++ b/scripts/production-release-drill.sh @@ -49,6 +49,7 @@ write_old_env() { "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-production-${current_commit:0:12}" \ "POSTGIS_IMAGE=who-need-help:postgis-production-${current_commit:0:12}" \ "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ + 'DATABASE_URL=ecto://release-drill:release-drill@database/release-drill' \ >"$fixture/.env" } write_old_env @@ -143,6 +144,10 @@ case "$*" in : >"$MOCK_FAIL_APP_MARKER" exit 23 fi + if [ "${MOCK_FAIL_RECOVERY:-}" = true ] && + grep -q "^APP_IMAGE=who-need-help:production-${MOCK_CURRENT_COMMIT%${MOCK_CURRENT_COMMIT#????????????}}$" "$env_file"; then + exit 24 + fi exit 0 ;; esac @@ -181,6 +186,10 @@ POLICY printf '%s\n' "$MOCK_TARGET_COMMIT" >"$MOCK_GIT_STATE" exit 0 ;; + *" checkout --detach $MOCK_CURRENT_COMMIT "*) + printf '%s\n' "$MOCK_CURRENT_COMMIT" >"$MOCK_GIT_STATE" + exit 0 + ;; esac printf 'Unexpected git invocation: %s\n' "$*" >&2 exit 1 @@ -195,7 +204,8 @@ if [ "$1" = inspect ]; then '{{.State.Status}}') printf 'running\n' ;; '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') printf 'healthy\n' ;; '{{.Config.Image}}') - if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ]; then + if [ "${MOCK_WRONG_RUNTIME_IMAGE:-}" = true ] && + [ "$(cat "$MOCK_GIT_STATE")" = "$MOCK_TARGET_COMMIT" ]; then printf 'who-need-help:production-wrong\n' else awk -F= '$1 == "APP_IMAGE" {print substr($0, index($0, "=") + 1)}' \ @@ -230,11 +240,22 @@ for command in curl jq pg_restore; do printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" done +install -m 755 /dev/null "$mock_bin/flock" +cat >"$mock_bin/flock" <<'EOF' +#!/bin/sh +set -eu +if [ "${MOCK_LOCK_BUSY:-}" = true ]; then + exit 1 +fi +exec /usr/bin/flock "$@" +EOF + touch "$fixture/mock-commands.log" chmod 600 "$fixture/mock-commands.log" "$fixture/git-state" container_env=( --env "MOCK_TARGET_COMMIT=$target_commit" + --env "MOCK_CURRENT_COMMIT=$current_commit" --env "MOCK_GIT_STATE=$remote_root/git-state" --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" --env "MOCK_ENV_FILE=$remote_root/.env" @@ -326,7 +347,7 @@ if [[ "$wrong_runtime_status" -eq 0 ]]; then echo "Release drill accepted a container created from the wrong image." >&2 exit 1 fi -grep -F 'Candidate runtime selected an unexpected image' \ +grep -F 'Production runtime does not use its approved image' \ "$run_dir/wrong-runtime-image.out" >/dev/null grep -F 'previous application will not be restarted' \ "$run_dir/wrong-runtime-image.out" >/dev/null @@ -380,7 +401,62 @@ grep -Fx "APP_IMAGE=who-need-help:production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null +grep -Fx "$current_commit" "$fixture/git-state" >/dev/null test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 2 +grep -R -F 'status=runtime-rolled-back' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null -echo "Isolated production release success/forward-only/safe-recovery drill passed." +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +rm -f "$fixture/app-up-failed" +set +e +run_release application_safe \ + --env MOCK_FAIL_APP_UP=once \ + --env MOCK_FAIL_RECOVERY=true \ + --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ + >"$run_dir/rollback-failure.out" 2>&1 +rollback_failure_status=$? +set -e +if [[ "$rollback_failure_status" -eq 0 ]]; then + echo "Production drill did not surface a failed automatic rollback." >&2 + exit 1 +fi +grep -F 'Automatic production rollback failed' \ + "$run_dir/rollback-failure.out" >/dev/null +grep -R -F 'status=rollback-failed' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null + +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +set +e +run_release application_safe --env MOCK_LOCK_BUSY=true \ + >"$run_dir/lock-busy.out" 2>&1 +lock_status=$? +set -e +if [[ "$lock_status" -eq 0 ]]; then + echo "Production drill did not reject a concurrent release lock." >&2 + exit 1 +fi +grep -F 'Another production release already holds' \ + "$run_dir/lock-busy.out" >/dev/null +test ! -s "$fixture/mock-commands.log" + +write_old_env +printf 'DEPLOYMENT_ENV=production\n' >>"$fixture/.env" +: >"$fixture/mock-commands.log" +set +e +run_release application_safe >"$run_dir/duplicate-env.out" 2>&1 +duplicate_status=$? +set -e +if [[ "$duplicate_status" -eq 0 ]]; then + echo "Production drill accepted a duplicate critical environment key." >&2 + exit 1 +fi +grep -F 'Expected exactly one DEPLOYMENT_ENV entry' \ + "$run_dir/duplicate-env.out" >/dev/null +test ! -s "$fixture/mock-commands.log" + +echo "Isolated production release success/failure/rollback/lock/env drill passed." diff --git a/scripts/production-release-remote.sh b/scripts/production-release-remote.sh index 53d4544..dcead32 100755 --- a/scripts/production-release-remote.sh +++ b/scripts/production-release-remote.sh @@ -28,7 +28,7 @@ if [[ "$root" != "/srv/who_need_help-production" ]]; then exit 2 fi -for command in curl docker git gzip jq pg_restore sha256sum; do +for command in curl docker flock git gzip jq pg_restore sha256sum; do command -v "$command" >/dev/null 2>&1 || { echo "Required production command is unavailable: $command" >&2 exit 2 @@ -45,14 +45,27 @@ read_value() { local key=$1 awk -v key="$key" ' index($0, key "=") == 1 { - print substr($0, length(key) + 2) - found = 1 - exit + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1) { + printf "Expected exactly one %s entry in the production environment; found %d.\n", key, count > "/dev/stderr" + exit 1 + } + print value } - END { if (!found) exit 1 } ' "$env_file" } +critical_env_keys=( + DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY + PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE DATABASE_URL +) +for critical_key in "${critical_env_keys[@]}"; do + read_value "$critical_key" >/dev/null +done + deployment_environment=$(read_value DEPLOYMENT_ENV) compose_project=$(read_value COMPOSE_PROJECT_NAME) database_mode=$(read_value DATABASE_MODE) @@ -90,28 +103,43 @@ current_commit=$(git -C "$root" rev-parse --verify HEAD) "$root/scripts/compose.sh" "$env_file" config --quiet -case "$app_topology" in - compact) expected_services=(app) ;; - split) expected_services=(web worker) ;; - *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;; -esac +[[ "$app_topology" == compact ]] || { + echo "The verified single-server production release workflow requires APP_TOPOLOGY=compact." >&2 + exit 2 +} +expected_services=(app) +runtime_services=(app) + +verify_service_image() { + local service=$1 expected_image=$2 expected_image_id=$3 + local container configured_image running_image_id state health -for service in "${expected_services[@]}"; do mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") [[ ${#containers[@]} -gt 0 ]] || { - echo "Production service is not running: $service" >&2 - exit 2 + echo "Production runtime service has no container: $service" >&2 + return 1 } for container in "${containers[@]}"; do state=$(docker inspect --format '{{.State.Status}}' "$container") health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") + running_image_id=$(docker inspect --format '{{.Image}}' "$container") [[ "$state" == "running" && "$health" == "healthy" ]] || { - echo "Production container is not healthy: $service" >&2 - exit 2 + echo "Production runtime container is not healthy: $service" >&2 + return 1 + } + [[ "$configured_image" == "$expected_image" && + "$running_image_id" == "$expected_image_id" ]] || { + echo "Production runtime does not use its approved image: $service" >&2 + return 1 } done -done +} + +current_app_image=$(read_value APP_IMAGE) +current_app_image_id=$(docker image inspect --format '{{.Id}}' "$current_app_image") +verify_service_image app "$current_app_image" "$current_app_image_id" curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null @@ -171,6 +199,13 @@ grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null git -C "$root" bundle verify "$bundle" >/dev/null +exec {release_lock_fd}>"$root/.git/wnh-production-release.lock" +chmod 600 "$root/.git/wnh-production-release.lock" +flock -n "$release_lock_fd" || { + echo "Another production release already holds $root/.git/wnh-production-release.lock." >&2 + exit 1 +} + bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') [[ "$bundle_head" == "$target_commit" ]] || { echo "Bundle HEAD does not match the approved target commit." >&2 @@ -189,13 +224,25 @@ git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { } policy_script=$(mktemp) -trap 'rm -f "$policy_script"' EXIT HUP INT TERM +# Invoked by the EXIT/HUP/INT/TERM traps below. +# shellcheck disable=SC2329 +cleanup_policy_script() { + local status=$? + trap - EXIT HUP INT TERM + rm -f "$policy_script" + exit "$status" +} +trap cleanup_policy_script EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script" chmod 600 "$policy_script" migration_policy_output=$( bash "$policy_script" "$current_commit" "$target_commit" "$root" ) rm -f "$policy_script" +unset -f cleanup_policy_script trap - EXIT HUP INT TERM printf '%s\n' "$migration_policy_output" migration_policy=$( @@ -216,14 +263,18 @@ fi release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}" release_dir="$root/output/releases/$release_id" -mkdir -p "$release_dir" -chmod 700 "$root/output" "$root/output/releases" "$release_dir" +[[ -d "$root/output/releases" ]] || { + echo "Production release output directory is missing: $root/output/releases" >&2 + exit 2 +} +install -d -m 700 "$release_dir" rollback_manifest="$release_dir/rollback-manifest.txt" { printf 'previous_commit=%s\n' "$current_commit" printf 'target_commit=%s\n' "$target_commit" printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)" + printf 'APP_IMAGE_ID=%s\n' "$current_app_image_id" printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)" printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)" printf 'migration_policy=%s\n' "$migration_policy" @@ -261,11 +312,14 @@ restore_image_revision() { mv "$temporary" "$env_file" chmod 600 "$env_file" + git -C "$root" checkout --detach "$current_commit" >/dev/null } rollback_runtime() { local status=$? + local rollback_ok=false trap - EXIT HUP INT TERM + set +e if [[ "$status" -ne 0 && "$revision_changed" == true && "$migration_policy" == "forward_only" && "$migration_started" == true ]]; then @@ -279,92 +333,55 @@ rollback_runtime() { echo "The previous application will not be restarted against a potentially incompatible schema." >&2 elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then echo "Release failed after selecting new images; restoring the previous immutable image tags." >&2 - restore_image_revision - "$root/scripts/compose.sh" "$env_file" \ - up -d --no-deps --no-build --force-recreate --wait \ - "${runtime_services[@]}" || true + previous_app_image=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") + previous_app_image_id=$(awk -F= '$1 == "APP_IMAGE_ID" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") - curl --fail --silent --show-error --max-time 15 \ - "https://$expected_domain/healthz/ready" >/dev/null || true - { - printf 'status=runtime-rolled-back\n' - printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - printf 'database_note=migrations_may_have_applied_and_were_not_reversed\n' - } >>"$rollback_manifest" - echo "The Git checkout and any applied migrations were intentionally not reversed automatically." >&2 + if restore_image_revision && + "$root/scripts/compose.sh" "$env_file" \ + up -d --no-deps --no-build --force-recreate --wait \ + "${runtime_services[@]}" && + verify_service_image app "$previous_app_image" "$previous_app_image_id" && + curl --fail --silent --show-error --max-time 15 \ + "https://$expected_domain/healthz/ready" >/dev/null; then + rollback_ok=true + fi + + if [[ "$rollback_ok" == true ]]; then + { + printf 'status=runtime-rolled-back\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'database_note=migrations_may_have_applied_and_were_not_reversed\n' + } >>"$rollback_manifest" + else + { + printf 'status=rollback-failed\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >>"$rollback_manifest" + echo "Automatic production rollback failed; inspect the rollback manifest and runtime before retrying." >&2 + fi fi exit "$status" } -case "$app_topology" in - compact) runtime_services=(app) ;; - split) runtime_services=(docker-api-proxy proxy web worker) ;; -esac - -verify_candidate_runtime() { - local expected_app_image expected_app_image_id service container - local configured_image running_image_id state health - - expected_app_image=$(read_value APP_IMAGE) - expected_app_image_id=$( - docker image inspect --format '{{.Id}}' "$expected_app_image" - ) - - for service in "${expected_services[@]}"; do - mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") - [[ ${#containers[@]} -gt 0 ]] || { - echo "Candidate runtime service has no container: $service" >&2 - return 1 - } - - for container in "${containers[@]}"; do - state=$(docker inspect --format '{{.State.Status}}' "$container") - health=$( - docker inspect \ - --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ - "$container" - ) - configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") - running_image_id=$(docker inspect --format '{{.Image}}' "$container") - - [[ "$state" == "running" && "$health" == "healthy" ]] || { - echo "Candidate runtime container is not healthy: $service" >&2 - return 1 - } - [[ "$configured_image" == "$expected_app_image" ]] || { - echo "Candidate runtime selected an unexpected image: $service" >&2 - return 1 - } - [[ "$running_image_id" == "$expected_app_image_id" ]] || { - echo "Candidate runtime image ID does not match the selected immutable image: $service" >&2 - return 1 - } - done - done -} - -trap rollback_runtime EXIT HUP INT TERM +trap rollback_runtime EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM if [[ "$branch" == "main" ]]; then git -C "$root" merge --ff-only "$release_ref" else git -C "$root" checkout --detach "$release_ref" fi - -"$root/scripts/set-deployment-revision.sh" "$env_file" --allow-pre-play revision_changed=true +"$root/scripts/set-deployment-revision.sh" "$env_file" --allow-pre-play "$root/scripts/validate-production-env.sh" \ "$env_file" "$expected_domain" --allow-pre-play "$root/scripts/check-environment-readiness.sh" \ "$env_file" --require-server-release expected_images=("$(read_value APP_IMAGE)") -if [[ "$app_topology" == split ]]; then - expected_images+=( - "$(read_value SOCKET_PROXY_IMAGE)" - "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" - ) -fi +declare -A approved_image_ids=() grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}" @@ -385,6 +402,7 @@ for image in "${expected_images[@]}"; do echo "Image manifest is missing the expected image ID: $image" >&2 exit 2 } + approved_image_ids["$image"]=$expected_id [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || { echo "Loaded image ID does not match the signed manifest: $image" >&2 exit 2 @@ -406,7 +424,9 @@ migration_started=true "$root/scripts/check-database.sh" "$env_file" &2 + exit 2 +} +install -d -m 700 "$local_backup_dir" scp -p \ "$ssh_target:$remote_backup" \ "$ssh_target:$remote_backup.sha256" \ diff --git a/scripts/test-release-drill.sh b/scripts/test-release-drill.sh index 68b5f6e..4a13371 100755 --- a/scripts/test-release-drill.sh +++ b/scripts/test-release-drill.sh @@ -48,6 +48,10 @@ write_old_env() { "APP_IMAGE=who-need-help:test-${current_commit:0:12}" \ "SOCKET_PROXY_IMAGE=who-need-help:socket-proxy-test-${current_commit:0:12}" \ "POSTGIS_IMAGE=who-need-help:postgis-test-${current_commit:0:12}" \ + 'POSTGRES_DB=who_need_help_test' \ + 'POSTGRES_USER=who_need_help_test' \ + 'POSTGRES_PASSWORD=test-release-drill-password' \ + 'DATABASE_URL=ecto://who_need_help_test:test-release-drill-password@db/who_need_help_test' \ >"$fixture/.env" } write_old_env @@ -150,6 +154,10 @@ case "$*" in : >"$MOCK_FAIL_APP_MARKER" exit 23 fi + if [ "${MOCK_FAIL_RECOVERY:-}" = true ] && + grep -q "^APP_IMAGE=who-need-help:test-${MOCK_CURRENT_COMMIT%${MOCK_CURRENT_COMMIT#????????????}}$" "$env_file"; then + exit 24 + fi exit 0 ;; esac @@ -251,6 +259,16 @@ for command in curl jq pg_restore; do printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" done +install -m 755 /dev/null "$mock_bin/flock" +cat >"$mock_bin/flock" <<'EOF' +#!/bin/sh +set -eu +if [ "${MOCK_LOCK_BUSY:-}" = true ]; then + exit 1 +fi +exec /usr/bin/flock "$@" +EOF + touch "$fixture/mock-commands.log" chmod 600 "$fixture/mock-commands.log" "$fixture/git-state" @@ -354,5 +372,57 @@ grep -Fx "APP_IMAGE=who-need-help:test-${current_commit:0:12}" "$fixture/.env" > grep -Fx "$current_commit" "$fixture/git-state" >/dev/null test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 2 +grep -R -F 'status=runtime-rolled-back' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null -echo "Isolated test release success/forward-only/safe-recovery drill passed." +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +rm -f "$fixture/app-up-failed" +set +e +run_release application_safe \ + --env MOCK_FAIL_APP_UP=once \ + --env MOCK_FAIL_RECOVERY=true \ + --env "MOCK_FAIL_APP_MARKER=$remote_root/app-up-failed" \ + >"$run_dir/rollback-failure.out" 2>&1 +rollback_status=$? +set -e +[[ "$rollback_status" -ne 0 ]] || { + echo "Test drill did not surface a failed automatic rollback." >&2 + exit 1 +} +grep -F 'Automatic test rollback failed' "$run_dir/rollback-failure.out" >/dev/null +grep -R -F 'status=rollback-failed' \ + "$fixture/output/releases" --include rollback-manifest.txt >/dev/null + +write_old_env +printf '%s\n' "$current_commit" >"$fixture/git-state" +: >"$fixture/mock-commands.log" +set +e +run_release application_safe --env MOCK_LOCK_BUSY=true \ + >"$run_dir/lock-busy.out" 2>&1 +lock_status=$? +set -e +[[ "$lock_status" -ne 0 ]] || { + echo "Test drill did not reject a concurrent release lock." >&2 + exit 1 +} +grep -F 'Another test release already holds' "$run_dir/lock-busy.out" >/dev/null +test ! -s "$fixture/mock-commands.log" + +write_old_env +printf 'DEPLOYMENT_ENV=test\n' >>"$fixture/.env" +: >"$fixture/mock-commands.log" +set +e +run_release application_safe >"$run_dir/duplicate-env.out" 2>&1 +duplicate_status=$? +set -e +[[ "$duplicate_status" -ne 0 ]] || { + echo "Test drill accepted a duplicate critical environment key." >&2 + exit 1 +} +grep -F 'Expected exactly one DEPLOYMENT_ENV entry' \ + "$run_dir/duplicate-env.out" >/dev/null +test ! -s "$fixture/mock-commands.log" + +echo "Isolated test release success/failure/rollback/lock/env drill passed." diff --git a/scripts/test-release-remote.sh b/scripts/test-release-remote.sh index 05fd188..891db04 100755 --- a/scripts/test-release-remote.sh +++ b/scripts/test-release-remote.sh @@ -22,7 +22,7 @@ case "$action" in *) usage; exit 2 ;; esac -for command in curl docker git gzip jq pg_restore realpath sha256sum; do +for command in curl docker flock git gzip jq pg_restore realpath sha256sum; do command -v "$command" >/dev/null 2>&1 || { echo "Required test release command is unavailable: $command" >&2 exit 2 @@ -45,14 +45,28 @@ read_value() { local key=$1 awk -v key="$key" ' index($0, key "=") == 1 { - print substr($0, length(key) + 2) - found = 1 - exit + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1) { + printf "Expected exactly one %s entry in the test environment; found %d.\n", key, count > "/dev/stderr" + exit 1 + } + print value } - END { if (!found) exit 1 } ' "$env_file" } +critical_env_keys=( + DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY + PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE + POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD DATABASE_URL +) +for critical_key in "${critical_env_keys[@]}"; do + read_value "$critical_key" >/dev/null +done + deployment_environment=$(read_value DEPLOYMENT_ENV) compose_project=$(read_value COMPOSE_PROJECT_NAME) database_mode=$(read_value DATABASE_MODE) @@ -85,33 +99,51 @@ current_commit=$(git -C "$root" rev-parse --verify HEAD) "$root/scripts/compose.sh" "$env_file" config --quiet -case "$app_topology" in - compact) - expected_services=(app) - runtime_services=(app) - ;; - split) - expected_services=(web worker) - runtime_services=(docker-api-proxy proxy web worker) - ;; - *) echo "Unexpected APP_TOPOLOGY." >&2; exit 2 ;; -esac +[[ "$app_topology" == compact ]] || { + echo "The verified single-server test release workflow requires APP_TOPOLOGY=compact." >&2 + exit 2 +} +expected_services=(app) +runtime_services=(app) + +verify_service_image() { + local service=$1 expected_image=$2 expected_image_id=$3 require_health=$4 + local container state health configured_image running_image_id -for service in db "${expected_services[@]}"; do mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") [[ ${#containers[@]} -gt 0 ]] || { - echo "Test service is not running: $service" >&2 - exit 2 + echo "Candidate test service has no container: $service" >&2 + return 1 } + for container in "${containers[@]}"; do state=$(docker inspect --format '{{.State.Status}}' "$container") health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") - [[ "$state" == running && "$health" == healthy ]] || { - echo "Test container is not healthy: $service" >&2 - exit 2 + configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") + running_image_id=$(docker inspect --format '{{.Image}}' "$container") + + [[ "$state" == running ]] || { + echo "Candidate test container is not running: $service" >&2 + return 1 + } + if [[ "$require_health" == true && "$health" != healthy ]]; then + echo "Candidate test container is not healthy: $service" >&2 + return 1 + fi + [[ "$configured_image" == "$expected_image" && + "$running_image_id" == "$expected_image_id" ]] || { + echo "Candidate test service does not use its approved image: $service" >&2 + return 1 } done -done +} + +current_app_image=$(read_value APP_IMAGE) +current_postgis_image=$(read_value POSTGIS_IMAGE) +current_app_image_id=$(docker image inspect --format '{{.Id}}' "$current_app_image") +current_postgis_image_id=$(docker image inspect --format '{{.Id}}' "$current_postgis_image") +verify_service_image app "$current_app_image" "$current_app_image_id" true +verify_service_image db "$current_postgis_image" "$current_postgis_image_id" true curl --fail --silent --show-error --max-time 15 \ "https://$expected_domain/healthz/ready" >/dev/null @@ -170,6 +202,13 @@ grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null git -C "$root" bundle verify "$bundle" >/dev/null +exec {release_lock_fd}>"$root/.git/wnh-test-release.lock" +chmod 600 "$root/.git/wnh-test-release.lock" +flock -n "$release_lock_fd" || { + echo "Another test release already holds $root/.git/wnh-test-release.lock." >&2 + exit 1 +} + bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') [[ "$bundle_head" == "$target_commit" ]] || { echo "Bundle HEAD does not match the approved test commit." >&2 @@ -188,11 +227,23 @@ git -C "$root" merge-base --is-ancestor "$current_commit" "$target_commit" || { } policy_script=$(mktemp) -trap 'rm -f "$policy_script"' EXIT HUP INT TERM +# Invoked by the EXIT/HUP/INT/TERM traps below. +# shellcheck disable=SC2329 +cleanup_policy_script() { + local status=$? + trap - EXIT HUP INT TERM + rm -f "$policy_script" + exit "$status" +} +trap cleanup_policy_script EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM git -C "$root" show "$release_ref:scripts/release-migration-policy.sh" >"$policy_script" chmod 700 "$policy_script" migration_policy_output=$(bash "$policy_script" "$current_commit" "$target_commit" "$root") rm -f "$policy_script" +unset -f cleanup_policy_script trap - EXIT HUP INT TERM printf '%s\n' "$migration_policy_output" migration_policy=$(awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output") @@ -211,15 +262,20 @@ fi release_id="$(date -u +%Y%m%dT%H%M%S%NZ)-${target_commit:0:12}" release_dir="$root/output/releases/$release_id" -mkdir -p "$release_dir" -chmod 700 "$root/output" "$root/output/releases" "$release_dir" +[[ -d "$root/output/releases" ]] || { + echo "Test release output directory is missing: $root/output/releases" >&2 + exit 2 +} +install -d -m 700 "$release_dir" rollback_manifest="$release_dir/rollback-manifest.txt" { printf 'previous_commit=%s\n' "$current_commit" printf 'target_commit=%s\n' "$target_commit" printf 'APP_IMAGE=%s\n' "$(read_value APP_IMAGE)" + printf 'APP_IMAGE_ID=%s\n' "$current_app_image_id" printf 'SOCKET_PROXY_IMAGE=%s\n' "$(read_value SOCKET_PROXY_IMAGE)" printf 'POSTGIS_IMAGE=%s\n' "$(read_value POSTGIS_IMAGE)" + printf 'POSTGIS_IMAGE_ID=%s\n' "$current_postgis_image_id" printf 'migration_policy=%s\n' "$migration_policy" printf 'database_backup=%s\n' "$backup" printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" @@ -256,7 +312,9 @@ restore_previous_revision() { rollback_runtime() { local status=$? + local rollback_ok=false trap - EXIT HUP INT TERM + set +e if [[ "$status" -ne 0 && "$revision_changed" == true && "$migration_policy" == forward_only && "$migration_started" == true ]]; then { @@ -267,33 +325,51 @@ rollback_runtime() { echo "Forward-only test release failed after migration started; automatic old-image restart is blocked." >&2 elif [[ "$status" -ne 0 && "$revision_changed" == true ]]; then echo "Test release failed before a forward-only schema boundary; restoring the previous revision." >&2 - restore_previous_revision - "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db || true - "$root/scripts/compose.sh" "$env_file" \ - up -d --no-deps --no-build --force-recreate --wait \ - "${runtime_services[@]}" || true - { - printf 'status=runtime-rolled-back\n' - printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - } >>"$rollback_manifest" + previous_app_image=$(awk -F= '$1 == "APP_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") + previous_app_image_id=$(awk -F= '$1 == "APP_IMAGE_ID" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") + previous_postgis_image=$(awk -F= '$1 == "POSTGIS_IMAGE" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") + previous_postgis_image_id=$(awk -F= '$1 == "POSTGIS_IMAGE_ID" {print substr($0,index($0,"=")+1)}' "$rollback_manifest") + + if restore_previous_revision && + "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db && + verify_service_image db "$previous_postgis_image" "$previous_postgis_image_id" true && + "$root/scripts/compose.sh" "$env_file" \ + up -d --no-deps --no-build --force-recreate --wait \ + "${runtime_services[@]}" && + verify_service_image app "$previous_app_image" "$previous_app_image_id" true && + curl --fail --silent --show-error --max-time 15 \ + "https://$expected_domain/healthz/ready" >/dev/null; then + rollback_ok=true + fi + + if [[ "$rollback_ok" == true ]]; then + { + printf 'status=runtime-rolled-back\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >>"$rollback_manifest" + else + { + printf 'status=rollback-failed\n' + printf 'failed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >>"$rollback_manifest" + echo "Automatic test rollback failed; inspect the rollback manifest and runtime before retrying." >&2 + fi fi exit "$status" } -trap rollback_runtime EXIT HUP INT TERM +trap rollback_runtime EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM gzip -dc "$image_archive" | docker load >/dev/null git -C "$root" checkout --detach "$release_ref" >/dev/null -"$root/scripts/set-deployment-revision.sh" "$env_file" revision_changed=true +"$root/scripts/set-deployment-revision.sh" "$env_file" "$root/scripts/validate-test-env.sh" "$env_file" "$expected_domain" expected_images=("$(read_value APP_IMAGE)" "$(read_value POSTGIS_IMAGE)") -if [[ "$app_topology" == split ]]; then - expected_images+=( - "$(read_value SOCKET_PROXY_IMAGE)" - "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" - ) -fi +declare -A approved_image_ids=() grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}" for image in "${expected_images[@]}"; do @@ -302,6 +378,7 @@ for image in "${expected_images[@]}"; do echo "Image manifest is missing the expected image: $image" >&2 exit 2 } + approved_image_ids["$image"]=$expected_id [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || { echo "Loaded test image ID does not match the manifest: $image" >&2 exit 2 @@ -320,41 +397,9 @@ if [[ "$migration_policy" == forward_only ]]; then fi "$root/scripts/compose.sh" "$env_file" up -d --no-build --wait db - -verify_service_image() { - local service=$1 expected_image=$2 require_health=$3 - local expected_image_id container state health configured_image running_image_id - - expected_image_id=$(docker image inspect --format '{{.Id}}' "$expected_image") - mapfile -t containers < <("$root/scripts/compose.sh" "$env_file" ps -q "$service") - [[ ${#containers[@]} -gt 0 ]] || { - echo "Candidate test service has no container: $service" >&2 - return 1 - } - - for container in "${containers[@]}"; do - state=$(docker inspect --format '{{.State.Status}}' "$container") - health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") - configured_image=$(docker inspect --format '{{.Config.Image}}' "$container") - running_image_id=$(docker inspect --format '{{.Image}}' "$container") - - [[ "$state" == running ]] || { - echo "Candidate test container is not running: $service" >&2 - return 1 - } - if [[ "$require_health" == true && "$health" != healthy ]]; then - echo "Candidate test container is not healthy: $service" >&2 - return 1 - fi - [[ "$configured_image" == "$expected_image" && - "$running_image_id" == "$expected_image_id" ]] || { - echo "Candidate test service does not use its approved image: $service" >&2 - return 1 - } - done -} - -verify_service_image db "$(read_value POSTGIS_IMAGE)" true +expected_postgis_image=$(read_value POSTGIS_IMAGE) +verify_service_image db "$expected_postgis_image" \ + "${approved_image_ids[$expected_postgis_image]}" true migration_started=true "$root/scripts/compose.sh" "$env_file" run --rm --no-deps --interactive=false migrate "$root/scripts/check-database.sh" "$env_file" &2 + exit 2 +} +install -d -m 700 "$local_backup_dir" scp -p \ "$ssh_target:$remote_backup" \ "$ssh_target:$remote_backup.sha256" \ diff --git a/test/scripts/production_release_artifact_root_test.py b/test/scripts/production_release_artifact_root_test.py index eab7b3d..1265a34 100644 --- a/test/scripts/production_release_artifact_root_test.py +++ b/test/scripts/production_release_artifact_root_test.py @@ -156,6 +156,7 @@ class ProductionReleaseArtifactRootTest(unittest.TestCase): release_dir = self.artifact_root / self.commit release_dir.mkdir(parents=True) + release_dir.chmod(0o700) archive = release_dir / f"who_need_help-{self.commit}-images-linux-amd64.tar.gz" with archive.open("wb") as output: with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed: @@ -209,6 +210,21 @@ class ProductionReleaseArtifactRootTest(unittest.TestCase): self.assertEqual(rejected.returncode, 2) self.assertIn("manifest commit does not match", rejected.stderr) + def test_existing_release_directory_with_broad_permissions_is_rejected(self): + release_dir = self.artifact_root / self.commit + release_dir.mkdir(parents=True) + release_dir.chmod(0o755) + + result = self.run_command( + [str(self.scripts / "prepare-production-release.sh")], + cwd=self.project, + env=self.artifact_env(), + check=False, + ) + + self.assertEqual(result.returncode, 2) + self.assertIn("owned mode-0700 directory", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/test/scripts/test_release_artifact_root_test.py b/test/scripts/test_release_artifact_root_test.py index 5d0f43e..8f8595b 100644 --- a/test/scripts/test_release_artifact_root_test.py +++ b/test/scripts/test_release_artifact_root_test.py @@ -85,6 +85,7 @@ class TestReleaseArtifactRootTest(unittest.TestCase): def write_existing_artifact(self): release_dir = self.artifact_root / self.commit release_dir.mkdir(parents=True) + release_dir.chmod(0o700) archive = ( release_dir / f"who_need_help-{self.commit}-test-images-linux-amd64.tar.gz" @@ -173,6 +174,21 @@ class TestReleaseArtifactRootTest(unittest.TestCase): self.assertEqual(result.returncode, 2) self.assertIn("dirty checkout", result.stderr) + def test_existing_release_directory_with_broad_permissions_is_rejected(self): + release_dir = self.artifact_root / self.commit + release_dir.mkdir(parents=True) + release_dir.chmod(0o755) + + result = self.run_command( + [str(self.scripts / "prepare-test-images.sh"), str(self.test_env)], + cwd=self.project, + env=self.environment(), + check=False, + ) + + self.assertEqual(result.returncode, 2) + self.assertIn("owned mode-0700 directory", result.stderr) + if __name__ == "__main__": unittest.main()