263 lines
9.1 KiB
Bash
Executable File
263 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
|
action=${1:-plan}
|
|
ssh_target=${2:-whoneedhelp}
|
|
remote_root=${WNH_PRODUCTION_REMOTE_ROOT:-/srv/who_need_help-production}
|
|
test_remote_root=${WNH_TEST_REMOTE_ROOT:-/srv/who_need_help-test}
|
|
expected_domain=${WNH_PRODUCTION_DOMAIN:-whoneedhelp.com}
|
|
expected_test_domain=${WNH_TEST_DOMAIN:-test.whoneedhelp.com}
|
|
|
|
case "$action" in
|
|
plan | prepare | apply) ;;
|
|
*)
|
|
echo "Usage: $0 [plan|prepare|apply] [SSH_TARGET]" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
for command in docker git gzip mktemp pg_restore realpath scp sha256sum ssh; do
|
|
command -v "$command" >/dev/null 2>&1 || {
|
|
echo "Required command is unavailable: $command" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
local_commit=$(git -C "$ROOT" rev-parse --verify HEAD)
|
|
[[ "$local_commit" =~ ^[0-9a-f]{40}$ ]] || {
|
|
echo "The local candidate is not a full 40-character commit SHA." >&2
|
|
exit 2
|
|
}
|
|
|
|
test_evidence=$(mktemp)
|
|
cleanup_test_evidence() {
|
|
rm -f "$test_evidence"
|
|
}
|
|
trap cleanup_test_evidence EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
remote_test_evidence="$test_remote_root/output/releases/verified/$local_commit.manifest"
|
|
ssh -o BatchMode=yes "$ssh_target" \
|
|
"test \"\$(stat -c '%a' '$remote_test_evidence')\" = 600 && cat '$remote_test_evidence'" \
|
|
>"$test_evidence" || {
|
|
echo "No mode-0600 successful test evidence exists for $local_commit." >&2
|
|
exit 2
|
|
}
|
|
|
|
require_test_evidence_value() {
|
|
local key=$1 expected=$2
|
|
awk -v key="$key" -v expected="$expected" '
|
|
index($0, key "=") == 1 {
|
|
value = substr($0, length(key) + 2)
|
|
count += 1
|
|
}
|
|
END {
|
|
if (count != 1 || value != expected) {
|
|
printf "Expected exactly one %s=%s entry in test evidence.\n", key, expected > "/dev/stderr"
|
|
exit 1
|
|
}
|
|
}
|
|
' "$test_evidence"
|
|
}
|
|
require_test_evidence_value format 1
|
|
require_test_evidence_value deployment test
|
|
require_test_evidence_value commit "$local_commit"
|
|
require_test_evidence_value domain "$expected_test_domain"
|
|
require_test_evidence_value status success
|
|
require_test_evidence_value public_health passed
|
|
echo "Verified exact-SHA test evidence: $remote_test_evidence"
|
|
|
|
remote_commit=$(
|
|
ssh -o BatchMode=yes "$ssh_target" \
|
|
"git -C '$remote_root' rev-parse --verify HEAD"
|
|
)
|
|
|
|
printf 'Local candidate commit: %s\n' "$local_commit"
|
|
printf 'Current production commit: %s\n' "$remote_commit"
|
|
|
|
if git -C "$ROOT" cat-file -e "$remote_commit^{commit}" 2>/dev/null; then
|
|
git -C "$ROOT" merge-base --is-ancestor "$remote_commit" "$local_commit" || {
|
|
echo "The local candidate is not a fast-forward from the production commit." >&2
|
|
exit 2
|
|
}
|
|
printf 'Pending commits: %s\n' \
|
|
"$(git -C "$ROOT" rev-list --count "$remote_commit..$local_commit")"
|
|
else
|
|
echo "The production commit is not present in the local object database." >&2
|
|
exit 2
|
|
fi
|
|
|
|
legacy_edge_paths=(
|
|
compose.edge.yaml
|
|
deploy/caddy/Caddyfile
|
|
)
|
|
if ! git -C "$ROOT" diff --quiet \
|
|
"$remote_commit" "$local_commit" -- "${legacy_edge_paths[@]}"; then
|
|
echo "The application release contains shared edge routing changes." >&2
|
|
echo "The shared edge serves both production and the test domain." >&2
|
|
echo "Move the approved route change through the independent server-edge workflow." >&2
|
|
exit 2
|
|
fi
|
|
echo "Shared edge routing files are unchanged; application release will not manage Caddy."
|
|
|
|
migration_policy_output=$(
|
|
"$ROOT/scripts/release-migration-policy.sh" "$remote_commit" "$local_commit"
|
|
)
|
|
printf '%s\n' "$migration_policy_output"
|
|
migration_policy=$(
|
|
awk -F= '$1 == "migration_policy" {print $2}' <<<"$migration_policy_output"
|
|
)
|
|
|
|
plan_failed=0
|
|
|
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
|
"WNH_PROJECT_ROOT='$remote_root' bash -s -- '$remote_root/.env' '$expected_domain' --allow-pre-play" \
|
|
<"$ROOT/scripts/validate-production-env.sh"; then
|
|
plan_failed=1
|
|
fi
|
|
|
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
|
"bash -s -- plan '$remote_root' '$expected_domain'" \
|
|
<"$ROOT/scripts/production-release-remote.sh"; then
|
|
plan_failed=1
|
|
fi
|
|
|
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
|
"bash -s -- '$remote_root/.env' --check-only production" \
|
|
<"$ROOT/scripts/backup-external-postgres.sh"; then
|
|
plan_failed=1
|
|
fi
|
|
|
|
if ! ssh -o BatchMode=yes "$ssh_target" \
|
|
"bash -s -- '$remote_root/.env' --require-server-release" \
|
|
<"$ROOT/scripts/check-environment-readiness.sh"; then
|
|
plan_failed=1
|
|
fi
|
|
|
|
if [[ "$plan_failed" -ne 0 ]]; then
|
|
echo "Production release plan has blocking checks; no remote state was changed." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$action" == "plan" ]]; then
|
|
echo "Production release plan passed; no remote state was changed."
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then
|
|
echo "Refusing to release a dirty checkout." >&2
|
|
exit 2
|
|
fi
|
|
|
|
confirmation="$expected_domain:$local_commit"
|
|
if [[ "$action" == "apply" ]]; then
|
|
if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$confirmation" ]]; then
|
|
echo "Release execution requires explicit approval in this exact process:" >&2
|
|
echo "WNH_PRODUCTION_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ "$migration_policy" == "forward_only" ]]; then
|
|
forward_confirmation="$expected_domain:$local_commit:forward-only"
|
|
if [[ "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" != "$forward_confirmation" ]]; then
|
|
echo "This release contains migrations that are not safe for an automatic old-image rollback." >&2
|
|
echo "A failed deployment after migration starts will keep the old application stopped." >&2
|
|
echo "Review the migration and recovery plan, then approve this exact boundary:" >&2
|
|
echo "WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$forward_confirmation \\" >&2
|
|
echo " WNH_PRODUCTION_RELEASE_CONFIRM=$confirmation $0 apply $ssh_target" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
"$ROOT/scripts/prepare-production-release.sh"
|
|
artifact_root=${WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT:-"$ROOT/output/releases"}
|
|
case "$artifact_root" in
|
|
/*) ;;
|
|
*)
|
|
echo "WNH_PRODUCTION_RELEASE_ARTIFACT_ROOT must be an absolute path." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
artifact_root=$(realpath --canonicalize-existing "$artifact_root")
|
|
release_dir="$artifact_root/$local_commit"
|
|
bundle="$release_dir/who_need_help-$local_commit.bundle"
|
|
image_archive="$release_dir/who_need_help-$local_commit-images-linux-amd64.tar.gz"
|
|
image_checksum="$image_archive.sha256"
|
|
image_manifest="$release_dir/who_need_help-$local_commit-images.manifest"
|
|
|
|
production_env=$(mktemp)
|
|
cleanup_production_env() {
|
|
rm -f "$production_env"
|
|
cleanup_test_evidence
|
|
}
|
|
trap cleanup_production_env EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
scp -p "$ssh_target:$remote_root/.env" "$production_env"
|
|
chmod 600 "$production_env"
|
|
"$ROOT/scripts/prepare-production-images.sh" "$production_env"
|
|
rm -f "$production_env"
|
|
trap cleanup_test_evidence EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
if [[ "$action" == "prepare" ]]; then
|
|
echo "Production release artifacts are prepared and verified locally; no remote state was changed."
|
|
exit 0
|
|
fi
|
|
|
|
remote_release_dir="$remote_root/output/releases/incoming"
|
|
remote_bundle="$remote_release_dir/$(basename -- "$bundle")"
|
|
remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")"
|
|
remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")"
|
|
remote_test_evidence="$remote_release_dir/test-verification-$local_commit.manifest"
|
|
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
|
|
remote_backup="$remote_root/output/backups/production/pre-$timestamp-${local_commit:0:12}.dump"
|
|
|
|
ssh -o BatchMode=yes "$ssh_target" \
|
|
"install -d -m 700 '$remote_release_dir'"
|
|
scp -p \
|
|
"$bundle" "$bundle.sha256" \
|
|
"$image_archive" "$image_checksum" "$image_manifest" \
|
|
"$ssh_target:$remote_release_dir/"
|
|
scp -p "$test_evidence" "$ssh_target:$remote_test_evidence"
|
|
|
|
ssh -o BatchMode=yes "$ssh_target" \
|
|
"bash -s -- '$remote_root/.env' '$remote_backup' production" \
|
|
<"$ROOT/scripts/backup-external-postgres.sh"
|
|
|
|
local_backup_dir="$ROOT/output/production-backups/$timestamp-${local_commit:0:12}"
|
|
mkdir -p "$ROOT/output/production-backups"
|
|
[[ ! -e "$local_backup_dir" ]] || {
|
|
echo "Local production backup directory already exists: $local_backup_dir" >&2
|
|
exit 2
|
|
}
|
|
install -d -m 700 "$local_backup_dir"
|
|
scp -p \
|
|
"$ssh_target:$remote_backup" \
|
|
"$ssh_target:$remote_backup.sha256" \
|
|
"$ssh_target:$remote_backup.metadata" \
|
|
"$local_backup_dir/"
|
|
(
|
|
cd "$local_backup_dir"
|
|
sha256sum --check "$(basename -- "$remote_backup.sha256")" >/dev/null
|
|
)
|
|
pg_restore --list "$local_backup_dir/$(basename -- "$remote_backup")" >/dev/null
|
|
echo "Copied and independently verified the pre-release backup outside the production server."
|
|
|
|
quoted_confirmation=$(printf '%q' "$confirmation")
|
|
quoted_forward_confirmation=$(
|
|
printf '%q' "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}"
|
|
)
|
|
ssh -o BatchMode=yes "$ssh_target" \
|
|
"WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest' '$remote_test_evidence'" \
|
|
<"$ROOT/scripts/production-release-remote.sh"
|
|
|
|
echo "Production release and public health verification completed."
|