From 89ac07ec20558d09ba973ccd426106ee64f37b90 Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Fri, 28 Aug 2026 02:26:22 +0300 Subject: [PATCH] Require exact test evidence for production releases --- docs/operations.md | 18 ++++++++ scripts/production-release-drill.sh | 54 ++++++++++++++++++++++- scripts/production-release-remote.sh | 29 ++++++++++-- scripts/production-release.sh | 57 +++++++++++++++++++++++- scripts/test-release-drill.sh | 10 +++++ scripts/test-release-remote.sh | 66 ++++++++++++++++++++++++++++ 6 files changed, 228 insertions(+), 6 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 01de3a2..381cb54 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -400,6 +400,16 @@ Do not use `deploy-up.sh` for an ordinary public test update on the small server: that command intentionally includes `--build` and therefore builds the release on the target host. +Only after the candidate database, runtime, cluster, image IDs, and public +readiness have all passed, the test release atomically records a mode-`0600` +manifest at +`/srv/who_need_help-test/output/releases/verified/.manifest`. +The filename and manifest both contain the full 40-character commit SHA. The +file also records the test domain, successful public-health result, topology, +immutable application/PostGIS image IDs, and migration policy. A failed test +release never creates this promotion evidence, and an existing manifest for +the same SHA is validated rather than overwritten. + Test always uses its own `who_need_help_test` PostGIS container/volume. Local development can use Mailpit; a public test deployment must use its own SMTP password and a visibly test-specific sender identity. @@ -1395,6 +1405,14 @@ allows only the absent Play App Signing certificate; the stricter publishing Android through Google Play. The plan neither uploads a bundle nor creates a backup. +Before those production checks begin, the workflow downloads the mode-`0600` +test-verification manifest whose filename exactly matches the local full SHA. +It rejects a missing manifest, another commit, another domain, a failed status, +or a failed public-health result. During `apply`, the same evidence file is +copied with the incoming artifacts and validated again before the production +checkout, database, images, or containers are changed. There is no "latest +successful" fallback and no prefix matching. + The verified single-server production workflow requires `APP_TOPOLOGY=compact`. A nonblocking lock at `.git/wnh-production-release.lock` rejects overlapping production releases diff --git a/scripts/production-release-drill.sh b/scripts/production-release-drill.sh index c70da46..4e3d158 100755 --- a/scripts/production-release-drill.sh +++ b/scripts/production-release-drill.sh @@ -81,6 +81,21 @@ printf '%s\n' \ 'image_count=1' \ "image=who-need-help:production-${target_commit:0:12}|$fixture_image_id" \ >"$image_manifest" +test_evidence="$fixture/output/releases/incoming/test-verification-$target_commit.manifest" +write_test_evidence() { + local commit=${1:-$target_commit} + local status=${2:-success} + printf '%s\n' \ + 'format=1' \ + 'deployment=test' \ + "commit=$commit" \ + 'domain=test.whoneedhelp.com' \ + "status=$status" \ + 'public_health=passed' \ + >"$test_evidence" + chmod 600 "$test_evidence" +} +write_test_evidence backup="$fixture/output/backups/production/pre-release.dump" printf 'isolated release drill backup\n' >"$backup" backup_hash=$(sha256sum "$backup" | awk '{print $1}') @@ -292,9 +307,46 @@ run_release() { "$remote_root/output/backups/production/$(basename -- "$backup")" \ "$policy" \ "$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \ - "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")" + "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")" \ + "$remote_root/output/releases/incoming/$(basename -- "$test_evidence")" } +rm -f "$test_evidence" +set +e +run_release application_safe >"$run_dir/missing-test-evidence.out" 2>&1 +missing_evidence_status=$? +set -e +[[ "$missing_evidence_status" -ne 0 ]] || { + echo "Production drill accepted missing test evidence." >&2 + exit 1 +} +grep -F 'Required release evidence is missing' \ + "$run_dir/missing-test-evidence.out" >/dev/null +test ! -s "$fixture/mock-commands.log" + +write_test_evidence 3333333333333333333333333333333333333333 +set +e +run_release application_safe >"$run_dir/wrong-test-commit.out" 2>&1 +wrong_evidence_status=$? +set -e +[[ "$wrong_evidence_status" -ne 0 ]] || { + echo "Production drill accepted test evidence for another commit." >&2 + exit 1 +} +test ! -s "$fixture/mock-commands.log" + +write_test_evidence "$target_commit" failed +set +e +run_release application_safe >"$run_dir/failed-test-status.out" 2>&1 +failed_evidence_status=$? +set -e +[[ "$failed_evidence_status" -ne 0 ]] || { + echo "Production drill accepted failed test evidence." >&2 + exit 1 +} +test ! -s "$fixture/mock-commands.log" + +write_test_evidence run_release forward_only >"$run_dir/forward-success.out" grep -Fx \ "image=who-need-help:production-${target_commit:0:12}|$fixture_image_id" \ diff --git a/scripts/production-release-remote.sh b/scripts/production-release-remote.sh index dcead32..7bd6ec9 100755 --- a/scripts/production-release-remote.sh +++ b/scripts/production-release-remote.sh @@ -11,10 +11,11 @@ backup=${6:-} expected_migration_policy=${7:-} image_archive=${8:-} image_manifest=${9:-} +test_evidence=${10:-} usage() { echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2 - echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2 + echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST TEST_EVIDENCE" >&2 } case "$action" in @@ -58,6 +59,22 @@ read_value() { ' "$env_file" } +require_manifest_value() { + local file=$1 key=$2 expected=$3 + awk -v key="$key" -v expected="$expected" ' + index($0, key "=") == 1 { + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1 || value != expected) { + printf "Expected exactly one %s=%s entry in %s.\n", key, expected, FILENAME > "/dev/stderr" + exit 1 + } + } + ' "$file" +} + critical_env_keys=( DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE DATABASE_URL @@ -160,7 +177,7 @@ fi if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" || -z "$expected_migration_policy" || -z "$image_archive" || - -z "$image_manifest" ]]; then + -z "$image_manifest" || -z "$test_evidence" ]]; then usage exit 2 fi @@ -173,7 +190,7 @@ fi for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \ "$backup.metadata" "$image_archive" "$image_archive.sha256" \ - "$image_manifest"; do + "$image_manifest" "$test_evidence"; do [[ -f "$required_file" ]] || { echo "Required release evidence is missing: $required_file" >&2 exit 2 @@ -197,6 +214,12 @@ gzip -t "$image_archive" grep -Fx 'format=1' "$image_manifest" >/dev/null grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null +require_manifest_value "$test_evidence" format 1 +require_manifest_value "$test_evidence" deployment test +require_manifest_value "$test_evidence" commit "$target_commit" +require_manifest_value "$test_evidence" domain test.whoneedhelp.com +require_manifest_value "$test_evidence" status success +require_manifest_value "$test_evidence" public_health passed git -C "$root" bundle verify "$bundle" >/dev/null exec {release_lock_fd}>"$root/.git/wnh-production-release.lock" diff --git a/scripts/production-release.sh b/scripts/production-release.sh index 7491c93..4888c10 100755 --- a/scripts/production-release.sh +++ b/scripts/production-release.sh @@ -5,7 +5,9 @@ ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) action=${1:-plan} ssh_target=${2:-whoneedhelp} remote_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production} +test_remote_root=${WNH_TEST_REMOTE_ROOT:-/srv/who_need_help-test} expected_domain=${WNH_PRODUCTION_DOMAIN:-whoneedhelp.com} +expected_test_domain=${WNH_TEST_DOMAIN:-test.whoneedhelp.com} case "$action" in plan | prepare | apply) ;; @@ -23,6 +25,51 @@ for command in docker git gzip mktemp pg_restore realpath scp sha256sum ssh; do done local_commit=$(git -C "$ROOT" rev-parse --verify HEAD) +[[ "$local_commit" =~ ^[0-9a-f]{40}$ ]] || { + echo "The local candidate is not a full 40-character commit SHA." >&2 + exit 2 +} + +test_evidence=$(mktemp) +cleanup_test_evidence() { + rm -f "$test_evidence" +} +trap cleanup_test_evidence EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +remote_test_evidence="$test_remote_root/output/releases/verified/$local_commit.manifest" +ssh -o BatchMode=yes "$ssh_target" \ + "test \"\$(stat -c '%a' '$remote_test_evidence')\" = 600 && cat '$remote_test_evidence'" \ + >"$test_evidence" || { + echo "No mode-0600 successful test evidence exists for $local_commit." >&2 + exit 2 + } + +require_test_evidence_value() { + local key=$1 expected=$2 + awk -v key="$key" -v expected="$expected" ' + index($0, key "=") == 1 { + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1 || value != expected) { + printf "Expected exactly one %s=%s entry in test evidence.\n", key, expected > "/dev/stderr" + exit 1 + } + } + ' "$test_evidence" +} +require_test_evidence_value format 1 +require_test_evidence_value deployment test +require_test_evidence_value commit "$local_commit" +require_test_evidence_value domain "$expected_test_domain" +require_test_evidence_value status success +require_test_evidence_value public_health passed +echo "Verified exact-SHA test evidence: $remote_test_evidence" + remote_commit=$( ssh -o BatchMode=yes "$ssh_target" \ "git -C '$remote_root' rev-parse --verify HEAD" @@ -145,6 +192,7 @@ image_manifest="$release_dir/who_need_help-$local_commit-images.manifest" production_env=$(mktemp) cleanup_production_env() { rm -f "$production_env" + cleanup_test_evidence } trap cleanup_production_env EXIT trap 'exit 129' HUP @@ -154,7 +202,10 @@ scp -p "$ssh_target:$remote_root/.env" "$production_env" chmod 600 "$production_env" "$ROOT/scripts/prepare-production-images.sh" "$production_env" rm -f "$production_env" -trap - EXIT HUP INT TERM +trap cleanup_test_evidence EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM if [[ "$action" == "prepare" ]]; then echo "Production release artifacts are prepared and verified locally; no remote state was changed." @@ -165,6 +216,7 @@ remote_release_dir="$remote_root/output/releases/incoming" remote_bundle="$remote_release_dir/$(basename -- "$bundle")" remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")" remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")" +remote_test_evidence="$remote_release_dir/test-verification-$local_commit.manifest" timestamp=$(date -u +%Y%m%dT%H%M%SZ) remote_backup="$remote_root/output/backups/production/pre-$timestamp-${local_commit:0:12}.dump" @@ -174,6 +226,7 @@ scp -p \ "$bundle" "$bundle.sha256" \ "$image_archive" "$image_checksum" "$image_manifest" \ "$ssh_target:$remote_release_dir/" +scp -p "$test_evidence" "$ssh_target:$remote_test_evidence" ssh -o BatchMode=yes "$ssh_target" \ "bash -s -- '$remote_root/.env' '$remote_backup' production" \ @@ -203,7 +256,7 @@ quoted_forward_confirmation=$( printf '%q' "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" ) ssh -o BatchMode=yes "$ssh_target" \ - "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \ + "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest' '$remote_test_evidence'" \ <"$ROOT/scripts/production-release-remote.sh" echo "Production release and public health verification completed." diff --git a/scripts/test-release-drill.sh b/scripts/test-release-drill.sh index 4a13371..de5e36c 100755 --- a/scripts/test-release-drill.sh +++ b/scripts/test-release-drill.sh @@ -331,6 +331,15 @@ if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then fi grep -R -F 'status=success' \ "$fixture/output/releases" --include rollback-manifest.txt >/dev/null +verified_manifest="$fixture/output/releases/verified/$target_commit.manifest" +test "$(stat -c '%a' "$verified_manifest")" = 600 +grep -Fx 'format=1' "$verified_manifest" >/dev/null +grep -Fx 'deployment=test' "$verified_manifest" >/dev/null +grep -Fx "commit=$target_commit" "$verified_manifest" >/dev/null +grep -Fx 'domain=test.whoneedhelp.com' "$verified_manifest" >/dev/null +grep -Fx 'status=success' "$verified_manifest" >/dev/null +grep -Fx 'public_health=passed' "$verified_manifest" >/dev/null +rm -f "$verified_manifest" write_old_env printf '%s\n' "$current_commit" >"$fixture/git-state" @@ -348,6 +357,7 @@ set -e exit 1 } grep -F 'automatic old-image restart is blocked' "$run_dir/forward-failure.out" >/dev/null +test ! -e "$verified_manifest" grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \ "$fixture/mock-commands.log")" = 1 diff --git a/scripts/test-release-remote.sh b/scripts/test-release-remote.sh index 891db04..e2ac89f 100755 --- a/scripts/test-release-remote.sh +++ b/scripts/test-release-remote.sh @@ -58,6 +58,22 @@ read_value() { ' "$env_file" } +require_manifest_value() { + local file=$1 key=$2 expected=$3 + awk -v key="$key" -v expected="$expected" ' + index($0, key "=") == 1 { + value = substr($0, length(key) + 2) + count += 1 + } + END { + if (count != 1 || value != expected) { + printf "Expected exactly one %s=%s entry in %s.\n", key, expected, FILENAME > "/dev/stderr" + exit 1 + } + } + ' "$file" +} + critical_env_keys=( DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE @@ -421,9 +437,59 @@ curl --fail --silent --show-error --max-time 30 \ printf 'status=success\n' printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } >>"$rollback_manifest" + +verified_dir="$root/output/releases/verified" +verified_manifest="$verified_dir/$target_commit.manifest" +install -d -m 700 "$verified_dir" + +if [[ -e "$verified_manifest" ]]; then + require_manifest_value "$verified_manifest" format 1 + require_manifest_value "$verified_manifest" deployment test + require_manifest_value "$verified_manifest" commit "$target_commit" + require_manifest_value "$verified_manifest" domain "$expected_domain" + require_manifest_value "$verified_manifest" status success + require_manifest_value "$verified_manifest" public_health passed + require_manifest_value "$verified_manifest" topology "$app_topology" + require_manifest_value "$verified_manifest" app_image "$(read_value APP_IMAGE)" + require_manifest_value "$verified_manifest" app_image_id \ + "${approved_image_ids[$expected_app_image]}" + require_manifest_value "$verified_manifest" postgis_image \ + "$(read_value POSTGIS_IMAGE)" + require_manifest_value "$verified_manifest" postgis_image_id \ + "${approved_image_ids[$expected_postgis_image]}" + require_manifest_value "$verified_manifest" migration_policy "$migration_policy" +else + verified_tmp=$(mktemp "$verified_dir/.${target_commit}.XXXXXX") + { + printf 'format=1\n' + printf 'deployment=test\n' + printf 'commit=%s\n' "$target_commit" + printf 'domain=%s\n' "$expected_domain" + printf 'status=success\n' + printf 'public_health=passed\n' + printf 'topology=%s\n' "$app_topology" + printf 'app_image=%s\n' "$(read_value APP_IMAGE)" + printf 'app_image_id=%s\n' "${approved_image_ids[$expected_app_image]}" + printf 'postgis_image=%s\n' "$(read_value POSTGIS_IMAGE)" + printf 'postgis_image_id=%s\n' \ + "${approved_image_ids[$expected_postgis_image]}" + printf 'migration_policy=%s\n' "$migration_policy" + printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >"$verified_tmp" + chmod 600 "$verified_tmp" + + if ! ln "$verified_tmp" "$verified_manifest" 2>/dev/null; then + rm -f "$verified_tmp" + echo "Concurrent test verification evidence already exists: $verified_manifest" >&2 + exit 1 + fi + rm -f "$verified_tmp" +fi + revision_changed=false trap - EXIT HUP INT TERM printf 'Test release completed: %s\n' "$target_commit" printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest" +printf 'Verified test evidence: %s\n' "$verified_manifest" printf 'Database backup: %s\n' "$backup"