Require exact test evidence for production releases

docs/operations.md

@@ -400,6 +400,16 @@ Do not use `deploy-up.sh` for an ordinary public test update on the small
 server: that command intentionally includes `--build` and therefore builds the
 release on the target host.
 
+Only after the candidate database, runtime, cluster, image IDs, and public
+readiness have all passed, the test release atomically records a mode-`0600`
+manifest at
+`/srv/who_need_help-test/output/releases/verified/<full-commit-sha>.manifest`.
+The filename and manifest both contain the full 40-character commit SHA. The
+file also records the test domain, successful public-health result, topology,
+immutable application/PostGIS image IDs, and migration policy. A failed test
+release never creates this promotion evidence, and an existing manifest for
+the same SHA is validated rather than overwritten.
+
 Test always uses its own `who_need_help_test` PostGIS container/volume. Local
 development can use Mailpit; a public test deployment must use its own SMTP
 password and a visibly test-specific sender identity.
@@ -1395,6 +1405,14 @@ allows only the absent Play App Signing certificate; the stricter
 publishing Android through Google Play. The plan neither uploads a bundle nor
 creates a backup.
 
+Before those production checks begin, the workflow downloads the mode-`0600`
+test-verification manifest whose filename exactly matches the local full SHA.
+It rejects a missing manifest, another commit, another domain, a failed status,
+or a failed public-health result. During `apply`, the same evidence file is
+copied with the incoming artifacts and validated again before the production
+checkout, database, images, or containers are changed. There is no "latest
+successful" fallback and no prefix matching.
+
 The verified single-server production workflow requires
 `APP_TOPOLOGY=compact`. A nonblocking lock at
 `.git/wnh-production-release.lock` rejects overlapping production releases

scripts/production-release-drill.sh

@@ -81,6 +81,21 @@ printf '%s\n' \
   'image_count=1' \
   "image=who-need-help:production-${target_commit:0:12}|$fixture_image_id" \
   >"$image_manifest"
+test_evidence="$fixture/output/releases/incoming/test-verification-$target_commit.manifest"
+write_test_evidence() {
+  local commit=${1:-$target_commit}
+  local status=${2:-success}
+  printf '%s\n' \
+    'format=1' \
+    'deployment=test' \
+    "commit=$commit" \
+    'domain=test.whoneedhelp.com' \
+    "status=$status" \
+    'public_health=passed' \
+    >"$test_evidence"
+  chmod 600 "$test_evidence"
+}
+write_test_evidence
 backup="$fixture/output/backups/production/pre-release.dump"
 printf 'isolated release drill backup\n' >"$backup"
 backup_hash=$(sha256sum "$backup" | awk '{print $1}')
@@ -292,9 +307,46 @@ run_release() {
       "$remote_root/output/backups/production/$(basename -- "$backup")" \
       "$policy" \
       "$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \
-      "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")"
+      "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")" \
+      "$remote_root/output/releases/incoming/$(basename -- "$test_evidence")"
 }
 
+rm -f "$test_evidence"
+set +e
+run_release application_safe >"$run_dir/missing-test-evidence.out" 2>&1
+missing_evidence_status=$?
+set -e
+[[ "$missing_evidence_status" -ne 0 ]] || {
+  echo "Production drill accepted missing test evidence." >&2
+  exit 1
+}
+grep -F 'Required release evidence is missing' \
+  "$run_dir/missing-test-evidence.out" >/dev/null
+test ! -s "$fixture/mock-commands.log"
+
+write_test_evidence 3333333333333333333333333333333333333333
+set +e
+run_release application_safe >"$run_dir/wrong-test-commit.out" 2>&1
+wrong_evidence_status=$?
+set -e
+[[ "$wrong_evidence_status" -ne 0 ]] || {
+  echo "Production drill accepted test evidence for another commit." >&2
+  exit 1
+}
+test ! -s "$fixture/mock-commands.log"
+
+write_test_evidence "$target_commit" failed
+set +e
+run_release application_safe >"$run_dir/failed-test-status.out" 2>&1
+failed_evidence_status=$?
+set -e
+[[ "$failed_evidence_status" -ne 0 ]] || {
+  echo "Production drill accepted failed test evidence." >&2
+  exit 1
+}
+test ! -s "$fixture/mock-commands.log"
+
+write_test_evidence
 run_release forward_only >"$run_dir/forward-success.out"
 grep -Fx \
   "image=who-need-help:production-${target_commit:0:12}|$fixture_image_id" \

scripts/production-release-remote.sh

@@ -11,10 +11,11 @@ backup=${6:-}
 expected_migration_policy=${7:-}
 image_archive=${8:-}
 image_manifest=${9:-}
+test_evidence=${10:-}
 
 usage() {
   echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2
-  echo "       $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2
+  echo "       $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST TEST_EVIDENCE" >&2
 }
 
 case "$action" in
@@ -58,6 +59,22 @@ read_value() {
   ' "$env_file"
 }
 
+require_manifest_value() {
+  local file=$1 key=$2 expected=$3
+  awk -v key="$key" -v expected="$expected" '
+    index($0, key "=") == 1 {
+      value = substr($0, length(key) + 2)
+      count += 1
+    }
+    END {
+      if (count != 1 || value != expected) {
+        printf "Expected exactly one %s=%s entry in %s.\n", key, expected, FILENAME > "/dev/stderr"
+        exit 1
+      }
+    }
+  ' "$file"
+}
+
 critical_env_keys=(
   DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY
   PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE DATABASE_URL
@@ -160,7 +177,7 @@ fi
 
 if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" ||
   -z "$expected_migration_policy" || -z "$image_archive" ||
-  -z "$image_manifest" ]]; then
+  -z "$image_manifest" || -z "$test_evidence" ]]; then
   usage
   exit 2
 fi
@@ -173,7 +190,7 @@ fi
 
 for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \
   "$backup.metadata" "$image_archive" "$image_archive.sha256" \
-  "$image_manifest"; do
+  "$image_manifest" "$test_evidence"; do
   [[ -f "$required_file" ]] || {
     echo "Required release evidence is missing: $required_file" >&2
     exit 2
@@ -197,6 +214,12 @@ gzip -t "$image_archive"
 grep -Fx 'format=1' "$image_manifest" >/dev/null
 grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null
 grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null
+require_manifest_value "$test_evidence" format 1
+require_manifest_value "$test_evidence" deployment test
+require_manifest_value "$test_evidence" commit "$target_commit"
+require_manifest_value "$test_evidence" domain test.whoneedhelp.com
+require_manifest_value "$test_evidence" status success
+require_manifest_value "$test_evidence" public_health passed
 git -C "$root" bundle verify "$bundle" >/dev/null
 
 exec {release_lock_fd}>"$root/.git/wnh-production-release.lock"

scripts/production-release.sh

@@ -5,7 +5,9 @@ ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
 action=${1:-plan}
 ssh_target=${2:-whoneedhelp}
 remote_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production}
+test_remote_root=${WNH_TEST_REMOTE_ROOT:-/srv/who_need_help-test}
 expected_domain=${WNH_PRODUCTION_DOMAIN:-whoneedhelp.com}
+expected_test_domain=${WNH_TEST_DOMAIN:-test.whoneedhelp.com}
 
 case "$action" in
   plan | prepare | apply) ;;
@@ -23,6 +25,51 @@ for command in docker git gzip mktemp pg_restore realpath scp sha256sum ssh; do
 done
 
 local_commit=$(git -C "$ROOT" rev-parse --verify HEAD)
+[[ "$local_commit" =~ ^[0-9a-f]{40}$ ]] || {
+  echo "The local candidate is not a full 40-character commit SHA." >&2
+  exit 2
+}
+
+test_evidence=$(mktemp)
+cleanup_test_evidence() {
+  rm -f "$test_evidence"
+}
+trap cleanup_test_evidence EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+remote_test_evidence="$test_remote_root/output/releases/verified/$local_commit.manifest"
+ssh -o BatchMode=yes "$ssh_target" \
+  "test \"\$(stat -c '%a' '$remote_test_evidence')\" = 600 && cat '$remote_test_evidence'" \
+  >"$test_evidence" || {
+    echo "No mode-0600 successful test evidence exists for $local_commit." >&2
+    exit 2
+  }
+
+require_test_evidence_value() {
+  local key=$1 expected=$2
+  awk -v key="$key" -v expected="$expected" '
+    index($0, key "=") == 1 {
+      value = substr($0, length(key) + 2)
+      count += 1
+    }
+    END {
+      if (count != 1 || value != expected) {
+        printf "Expected exactly one %s=%s entry in test evidence.\n", key, expected > "/dev/stderr"
+        exit 1
+      }
+    }
+  ' "$test_evidence"
+}
+require_test_evidence_value format 1
+require_test_evidence_value deployment test
+require_test_evidence_value commit "$local_commit"
+require_test_evidence_value domain "$expected_test_domain"
+require_test_evidence_value status success
+require_test_evidence_value public_health passed
+echo "Verified exact-SHA test evidence: $remote_test_evidence"
+
 remote_commit=$(
   ssh -o BatchMode=yes "$ssh_target" \
     "git -C '$remote_root' rev-parse --verify HEAD"
@@ -145,6 +192,7 @@ image_manifest="$release_dir/who_need_help-$local_commit-images.manifest"
 production_env=$(mktemp)
 cleanup_production_env() {
   rm -f "$production_env"
+  cleanup_test_evidence
 }
 trap cleanup_production_env EXIT
 trap 'exit 129' HUP
@@ -154,7 +202,10 @@ scp -p "$ssh_target:$remote_root/.env" "$production_env"
 chmod 600 "$production_env"
 "$ROOT/scripts/prepare-production-images.sh" "$production_env"
 rm -f "$production_env"
-trap - EXIT HUP INT TERM
+trap cleanup_test_evidence EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
 
 if [[ "$action" == "prepare" ]]; then
   echo "Production release artifacts are prepared and verified locally; no remote state was changed."
@@ -165,6 +216,7 @@ remote_release_dir="$remote_root/output/releases/incoming"
 remote_bundle="$remote_release_dir/$(basename -- "$bundle")"
 remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")"
 remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")"
+remote_test_evidence="$remote_release_dir/test-verification-$local_commit.manifest"
 timestamp=$(date -u +%Y%m%dT%H%M%SZ)
 remote_backup="$remote_root/output/backups/production/pre-$timestamp-${local_commit:0:12}.dump"
 
@@ -174,6 +226,7 @@ scp -p \
   "$bundle" "$bundle.sha256" \
   "$image_archive" "$image_checksum" "$image_manifest" \
   "$ssh_target:$remote_release_dir/"
+scp -p "$test_evidence" "$ssh_target:$remote_test_evidence"
 
 ssh -o BatchMode=yes "$ssh_target" \
   "bash -s -- '$remote_root/.env' '$remote_backup' production" \
@@ -203,7 +256,7 @@ quoted_forward_confirmation=$(
   printf '%q' "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}"
 )
 ssh -o BatchMode=yes "$ssh_target" \
-  "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \
+  "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest' '$remote_test_evidence'" \
   <"$ROOT/scripts/production-release-remote.sh"
 
 echo "Production release and public health verification completed."

scripts/test-release-drill.sh

@@ -331,6 +331,15 @@ if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then
 fi
 grep -R -F 'status=success' \
   "$fixture/output/releases" --include rollback-manifest.txt >/dev/null
+verified_manifest="$fixture/output/releases/verified/$target_commit.manifest"
+test "$(stat -c '%a' "$verified_manifest")" = 600
+grep -Fx 'format=1' "$verified_manifest" >/dev/null
+grep -Fx 'deployment=test' "$verified_manifest" >/dev/null
+grep -Fx "commit=$target_commit" "$verified_manifest" >/dev/null
+grep -Fx 'domain=test.whoneedhelp.com' "$verified_manifest" >/dev/null
+grep -Fx 'status=success' "$verified_manifest" >/dev/null
+grep -Fx 'public_health=passed' "$verified_manifest" >/dev/null
+rm -f "$verified_manifest"
 
 write_old_env
 printf '%s\n' "$current_commit" >"$fixture/git-state"
@@ -348,6 +357,7 @@ set -e
   exit 1
 }
 grep -F 'automatic old-image restart is blocked' "$run_dir/forward-failure.out" >/dev/null
+test ! -e "$verified_manifest"
 grep -Fx "APP_IMAGE=who-need-help:test-${target_commit:0:12}" "$fixture/.env" >/dev/null
 test "$(grep -Fc 'compose:up -d --no-deps --no-build --force-recreate --wait app' \
   "$fixture/mock-commands.log")" = 1

scripts/test-release-remote.sh

@@ -58,6 +58,22 @@ read_value() {
   ' "$env_file"
 }
 
+require_manifest_value() {
+  local file=$1 key=$2 expected=$3
+  awk -v key="$key" -v expected="$expected" '
+    index($0, key "=") == 1 {
+      value = substr($0, length(key) + 2)
+      count += 1
+    }
+    END {
+      if (count != 1 || value != expected) {
+        printf "Expected exactly one %s=%s entry in %s.\n", key, expected, FILENAME > "/dev/stderr"
+        exit 1
+      }
+    }
+  ' "$file"
+}
+
 critical_env_keys=(
   DEPLOYMENT_ENV COMPOSE_PROJECT_NAME DATABASE_MODE APP_TOPOLOGY
   PHX_HOST WNH_BASE_URL APP_IMAGE SOCKET_PROXY_IMAGE POSTGIS_IMAGE
@@ -421,9 +437,59 @@ curl --fail --silent --show-error --max-time 30 \
   printf 'status=success\n'
   printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 } >>"$rollback_manifest"
+
+verified_dir="$root/output/releases/verified"
+verified_manifest="$verified_dir/$target_commit.manifest"
+install -d -m 700 "$verified_dir"
+
+if [[ -e "$verified_manifest" ]]; then
+  require_manifest_value "$verified_manifest" format 1
+  require_manifest_value "$verified_manifest" deployment test
+  require_manifest_value "$verified_manifest" commit "$target_commit"
+  require_manifest_value "$verified_manifest" domain "$expected_domain"
+  require_manifest_value "$verified_manifest" status success
+  require_manifest_value "$verified_manifest" public_health passed
+  require_manifest_value "$verified_manifest" topology "$app_topology"
+  require_manifest_value "$verified_manifest" app_image "$(read_value APP_IMAGE)"
+  require_manifest_value "$verified_manifest" app_image_id \
+    "${approved_image_ids[$expected_app_image]}"
+  require_manifest_value "$verified_manifest" postgis_image \
+    "$(read_value POSTGIS_IMAGE)"
+  require_manifest_value "$verified_manifest" postgis_image_id \
+    "${approved_image_ids[$expected_postgis_image]}"
+  require_manifest_value "$verified_manifest" migration_policy "$migration_policy"
+else
+  verified_tmp=$(mktemp "$verified_dir/.${target_commit}.XXXXXX")
+  {
+    printf 'format=1\n'
+    printf 'deployment=test\n'
+    printf 'commit=%s\n' "$target_commit"
+    printf 'domain=%s\n' "$expected_domain"
+    printf 'status=success\n'
+    printf 'public_health=passed\n'
+    printf 'topology=%s\n' "$app_topology"
+    printf 'app_image=%s\n' "$(read_value APP_IMAGE)"
+    printf 'app_image_id=%s\n' "${approved_image_ids[$expected_app_image]}"
+    printf 'postgis_image=%s\n' "$(read_value POSTGIS_IMAGE)"
+    printf 'postgis_image_id=%s\n' \
+      "${approved_image_ids[$expected_postgis_image]}"
+    printf 'migration_policy=%s\n' "$migration_policy"
+    printf 'completed_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  } >"$verified_tmp"
+  chmod 600 "$verified_tmp"
+
+  if ! ln "$verified_tmp" "$verified_manifest" 2>/dev/null; then
+    rm -f "$verified_tmp"
+    echo "Concurrent test verification evidence already exists: $verified_manifest" >&2
+    exit 1
+  fi
+  rm -f "$verified_tmp"
+fi
+
 revision_changed=false
 trap - EXIT HUP INT TERM
 
 printf 'Test release completed: %s\n' "$target_commit"
 printf 'Rollback/runtime evidence: %s\n' "$rollback_manifest"
+printf 'Verified test evidence: %s\n' "$verified_manifest"
 printf 'Database backup: %s\n' "$backup"