diff --git a/docs/operations.md b/docs/operations.md index bdd7dcf..123ef10 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1077,17 +1077,24 @@ WNH_PRODUCTION_RELEASE_CONFIRM=whoneedhelp.com:FULL_COMMIT \ The apply path refuses tracked local or remote modifications. It then: 1. creates and verifies a full Git bundle for exactly that clean commit; -2. uploads only that bundle to the production checkout's ignored +2. reads the production environment over SSH into a mode-0600 temporary local + file, selects the candidate's immutable image tags, builds the required + `linux/amd64` images on the development workstation, and deletes that + temporary environment file; +3. records every image ID in a manifest, creates a timestamp-free + gzip-compressed Docker archive, verifies its SHA-256, and uploads the + bundle, archive, checksum, and manifest to the production checkout's ignored `output/releases/`; -3. creates a custom-format PostgreSQL 18 backup without exposing the database +4. creates a custom-format PostgreSQL 18 backup without exposing the database password in process arguments; -4. verifies its archive catalog and SHA-256, then copies and verifies the +5. verifies its archive catalog and SHA-256, then copies and verifies the backup again under local ignored `output/production-backups/`; -5. fast-forwards the production checkout without accessing or changing the +6. fast-forwards the production checkout without accessing or changing the test checkout or public remote; -6. selects immutable per-commit application image tags, applies migrations, - starts only the application topology, and verifies public readiness through - the already-running shared edge plus the Android App Links endpoints. +7. verifies the transferred archive and manifest, loads the ready images + without compiling on the production host, applies migrations, starts only + the application topology, and verifies public readiness through the + already-running shared edge plus the Android App Links endpoints. Every newly added migration must have one reviewed entry in `priv/repo/migration_application_compatibility.tsv`. `application_safe` means @@ -1101,12 +1108,13 @@ WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=whoneedhelp.com:FULL_COMMIT:forward-only \ ./scripts/production-release.sh apply whoneedhelp ``` -For a forward-only release, all candidate images are built first, the old -application is stopped before migration begins, and the target application is -started only after the migration runner succeeds. If anything fails after the -migration begins, the release deliberately leaves the old application stopped -and records that boundary in the release manifest. Restarting an older image -against a potentially incompatible schema is never automatic. +For a forward-only release, all candidate images are built and transferred +first, the old application is stopped before migration begins, and the target +application is started only after the migration runner succeeds. If anything +fails after the migration begins, the release deliberately leaves the old +application stopped and records that boundary in the release manifest. +Restarting an older image against a potentially incompatible schema is never +automatic. For an `application_safe` release, an application startup failure restores the previous immutable application image tags and attempts to recover public diff --git a/scripts/prepare-production-images.sh b/scripts/prepare-production-images.sh new file mode 100755 index 0000000..c6c110e --- /dev/null +++ b/scripts/prepare-production-images.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +source_env=${1:-} + +if [[ -z "$source_env" || ! -f "$source_env" ]]; then + echo "Usage: $0 PRODUCTION_ENV_FILE" >&2 + exit 2 +fi + +for command in docker gzip jq sha256sum; do + command -v "$command" >/dev/null 2>&1 || { + echo "Required command is unavailable: $command" >&2 + exit 2 + } +done + +if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then + echo "Refusing to build production images from a dirty tracked checkout." >&2 + exit 2 +fi + +commit=$(git -C "$ROOT" rev-parse --verify HEAD) +short_commit=${commit:0:12} +release_dir="$ROOT/output/releases/$commit" +archive="$release_dir/who_need_help-$commit-images-linux-amd64.tar.gz" +checksum="$archive.sha256" +manifest="$release_dir/who_need_help-$commit-images.manifest" + +mkdir -p "$release_dir" +chmod 700 "$ROOT/output" "$ROOT/output/releases" "$release_dir" + +build_env=$(mktemp "$release_dir/.production-image-build.XXXXXX") +cleanup() { + trap - EXIT HUP INT TERM + rm -f "$build_env" +} +trap cleanup EXIT HUP INT TERM +install -m 600 "$source_env" "$build_env" + +read_value() { + local key=$1 + awk -v key="$key" ' + index($0, key "=") == 1 { + print substr($0, length(key) + 2) + found = 1 + exit + } + END { if (!found) exit 1 } + ' "$build_env" +} + +replace_value() { + local key=$1 + local value=$2 + local temporary + temporary=$(mktemp "$release_dir/.production-image-env.XXXXXX") + chmod 600 "$temporary" + awk -v key="$key" -v value="$value" ' + index($0, key "=") == 1 { print key "=" value; found = 1; next } + { print } + END { if (!found) exit 1 } + ' "$build_env" >"$temporary" + mv "$temporary" "$build_env" + chmod 600 "$build_env" +} + +[[ "$(read_value DEPLOYMENT_ENV)" == production ]] || { + echo "The image build input is not a production environment." >&2 + exit 2 +} +[[ "$(read_value DATABASE_MODE)" == external ]] || { + echo "The verified production image workflow expects DATABASE_MODE=external." >&2 + exit 2 +} + +replace_value APP_IMAGE "who-need-help:production-$short_commit" +replace_value SOCKET_PROXY_IMAGE \ + "who-need-help:socket-proxy-production-$short_commit" +replace_value POSTGIS_IMAGE "who-need-help:postgis-production-$short_commit" + +topology=$(read_value APP_TOPOLOGY) +case "$topology" in + compact) + build_services=(migrate) + ;; + split) + build_services=(docker-api-proxy proxy migrate) + ;; + *) + echo "APP_TOPOLOGY must be compact or split." >&2 + exit 2 + ;; +esac + +app_image=$(read_value APP_IMAGE) +images=("$app_image") +if [[ "$topology" == split ]]; then + socket_proxy_image=$(read_value SOCKET_PROXY_IMAGE) + proxy_image=$( + "$ROOT/scripts/compose.sh" "$build_env" config --format json | + jq -er '.services.proxy.image' + ) + images+=("$socket_proxy_image" "$proxy_image") +fi + +if [[ -e "$archive" || -e "$checksum" || -e "$manifest" ]]; then + [[ -f "$archive" && -f "$checksum" && -f "$manifest" ]] || { + echo "The production image package is incomplete; refusing to overwrite it." >&2 + exit 2 + } + ( + cd "$release_dir" + sha256sum --check "$(basename -- "$checksum")" >/dev/null + ) + grep -Fx "commit=$commit" "$manifest" >/dev/null + echo "Production image package already exists and passed checksum verification." +else + "$ROOT/scripts/compose.sh" "$build_env" build "${build_services[@]}" + + manifest_tmp=$(mktemp "$release_dir/.production-images-manifest.XXXXXX") + archive_tmp=$(mktemp "$release_dir/.production-images-archive.XXXXXX") + trap 'rm -f "$build_env" "$manifest_tmp" "$archive_tmp"' EXIT HUP INT TERM + + { + printf 'format=1\n' + printf 'commit=%s\n' "$commit" + printf 'platform=linux/amd64\n' + printf 'topology=%s\n' "$topology" + printf 'image_count=%s\n' "${#images[@]}" + for image in "${images[@]}"; do + platform=$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image") + [[ "$platform" == linux/amd64 ]] || { + echo "Production image has an unexpected platform: $image ($platform)" >&2 + exit 2 + } + image_id=$(docker image inspect --format '{{.Id}}' "$image") + printf 'image=%s|%s\n' "$image" "$image_id" + done + } >"$manifest_tmp" + + docker save "${images[@]}" | gzip -n -9 >"$archive_tmp" + mv "$archive_tmp" "$archive" + mv "$manifest_tmp" "$manifest" + chmod 600 "$archive" "$manifest" + hash=$(sha256sum "$archive" | awk '{print $1}') + printf '%s %s\n' "$hash" "$(basename -- "$archive")" >"$checksum" + chmod 600 "$checksum" +fi + +( + cd "$release_dir" + sha256sum --check "$(basename -- "$checksum")" >/dev/null +) +gzip -t "$archive" +grep -Fx 'platform=linux/amd64' "$manifest" >/dev/null +grep -Fx "topology=$topology" "$manifest" >/dev/null +test "$(grep -c '^image=' "$manifest")" = "${#images[@]}" +for image in "${images[@]}"; do + awk -F'|' -v image="$image" ' + $1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 } + END { if (!found) exit 1 } + ' "$manifest" +done + +cleanup +printf 'Production image archive: %s\n' "$archive" +printf 'Image archive checksum: %s\n' "$checksum" +printf 'Image manifest: %s\n' "$manifest" diff --git a/scripts/production-release-drill.sh b/scripts/production-release-drill.sh index 1fe97df..9ad1cbd 100755 --- a/scripts/production-release-drill.sh +++ b/scripts/production-release-drill.sh @@ -13,10 +13,20 @@ current_commit=1111111111111111111111111111111111111111 target_commit=2222222222222222222222222222222222222222 release_confirmation="whoneedhelp.com:$target_commit" forward_confirmation="whoneedhelp.com:$target_commit:forward-only" +mock_candidate_id="sha256:$(printf 'who-need-help-release-drill-candidate' | sha256sum | awk '{print $1}')" cleanup() { + status=$? trap - EXIT HUP INT TERM + if [[ "$status" -ne 0 ]]; then + for output in "$run_dir"/*.out; do + [[ -f "$output" ]] || continue + printf '\n--- %s ---\n' "$(basename -- "$output")" >&2 + cat "$output" >&2 + done + fi find "$run_dir" -xdev -depth -delete 2>/dev/null || true + exit "$status" } trap cleanup EXIT HUP INT TERM @@ -49,12 +59,28 @@ bundle="$fixture/output/releases/incoming/who_need_help-$target_commit.bundle" printf 'isolated release drill bundle\n' >"$bundle" bundle_hash=$(sha256sum "$bundle" | awk '{print $1}') printf '%s %s\n' "$bundle_hash" "$(basename -- "$bundle")" >"$bundle.sha256" +image_archive="$fixture/output/releases/incoming/who_need_help-$target_commit-images-linux-amd64.tar.gz" +printf 'isolated release drill image archive\n' | gzip -n >"$image_archive" +image_hash=$(sha256sum "$image_archive" | awk '{print $1}') +printf '%s %s\n' "$image_hash" "$(basename -- "$image_archive")" \ + >"$image_archive.sha256" +image_manifest="$fixture/output/releases/incoming/who_need_help-$target_commit-images.manifest" +printf '%s\n' \ + 'format=1' \ + "commit=$target_commit" \ + 'platform=linux/amd64' \ + 'topology=compact' \ + 'image_count=1' \ + "image=who-need-help:production-${target_commit:0:12}|$mock_candidate_id" \ + >"$image_manifest" backup="$fixture/output/backups/production/pre-release.dump" printf 'isolated release drill backup\n' >"$backup" backup_hash=$(sha256sum "$backup" | awk '{print $1}') printf '%s %s\n' "$backup_hash" "$(basename -- "$backup")" >"$backup.sha256" printf 'environment=production\n' >"$backup.metadata" -chmod 600 "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata" +chmod 600 "$bundle" "$bundle.sha256" "$image_archive" \ + "$image_archive.sha256" "$image_manifest" "$backup" "$backup.sha256" \ + "$backup.metadata" for script in validate-production-env.sh check-environment-readiness.sh \ verify-realtime-cluster.sh verify-beam-runtime.sh; do @@ -86,10 +112,6 @@ shift case "$*" in 'config --quiet') exit 0 ;; 'ps -q app') printf 'app-1\n'; exit 0 ;; - 'build migrate') - printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" - exit 0 - ;; 'stop app') printf 'compose:%s\n' "$*" >>"$MOCK_COMMAND_LOG" exit 0 @@ -173,21 +195,30 @@ if [ "$1" = inspect ]; then "$MOCK_ENV_FILE" fi ;; - '{{.Image}}') printf 'sha256:mock-candidate\n' ;; + '{{.Image}}') printf '%s\n' "$MOCK_CANDIDATE_ID" ;; *) exit 1 ;; esac exit 0 fi if [ "$1" = image ] && [ "$2" = inspect ] && - [ "$3" = --format ] && [ "$4" = '{{.Id}}' ]; then - printf 'sha256:mock-candidate\n' + [ "$3" = --format ]; then + case "$4" in + '{{.Id}}') printf '%s\n' "$MOCK_CANDIDATE_ID" ;; + '{{.Os}}/{{.Architecture}}') printf 'linux/amd64\n' ;; + *) exit 1 ;; + esac + exit 0 +fi +if [ "$1" = load ]; then + cat >/dev/null + printf 'docker:load\n' >>"$MOCK_COMMAND_LOG" exit 0 fi printf 'Unexpected docker invocation: %s\n' "$*" >&2 exit 1 EOF -for command in curl pg_restore; do +for command in curl jq pg_restore; do install -m 755 /dev/null "$mock_bin/$command" printf '%s\n' '#!/bin/sh' 'exit 0' >"$mock_bin/$command" done @@ -200,6 +231,7 @@ container_env=( --env "MOCK_GIT_STATE=$remote_root/git-state" --env "MOCK_COMMAND_LOG=$remote_root/mock-commands.log" --env "MOCK_ENV_FILE=$remote_root/.env" + --env "MOCK_CANDIDATE_ID=$mock_candidate_id" --env "PATH=/mock-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ) container_mounts=( @@ -230,7 +262,9 @@ run_release() { "$remote_root/output/releases/incoming/$(basename -- "$bundle")" \ "$target_commit" \ "$remote_root/output/backups/production/$(basename -- "$backup")" \ - "$policy" + "$policy" \ + "$remote_root/output/releases/incoming/$(basename -- "$image_archive")" \ + "$remote_root/output/releases/incoming/$(basename -- "$image_manifest")" } run_release forward_only >"$run_dir/forward-success.out" @@ -238,7 +272,11 @@ grep -Fx "APP_IMAGE=who-need-help:production-${target_commit:0:12}" \ "$fixture/.env" >/dev/null grep -Fx "CADDY_IMAGE=who-need-help:caddy-production-${current_commit:0:12}" \ "$fixture/.env" >/dev/null -grep -F 'compose:build migrate' "$fixture/mock-commands.log" >/dev/null +grep -Fx 'docker:load' "$fixture/mock-commands.log" >/dev/null +if grep -F 'compose:build' "$fixture/mock-commands.log" >/dev/null; then + echo "Production release drill unexpectedly compiled on the production host." >&2 + exit 1 +fi grep -F 'compose:stop app' "$fixture/mock-commands.log" >/dev/null grep -F 'compose:run --rm --no-deps --interactive=false migrate' \ "$fixture/mock-commands.log" >/dev/null diff --git a/scripts/production-release-remote.sh b/scripts/production-release-remote.sh index bc27eb0..9ab970f 100755 --- a/scripts/production-release-remote.sh +++ b/scripts/production-release-remote.sh @@ -9,10 +9,12 @@ bundle=${4:-} target_commit=${5:-} backup=${6:-} expected_migration_policy=${7:-} +image_archive=${8:-} +image_manifest=${9:-} usage() { echo "Usage: $0 plan /srv/who_need_help-production whoneedhelp.com" >&2 - echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY" >&2 + echo " $0 apply /srv/who_need_help-production whoneedhelp.com BUNDLE COMMIT BACKUP MIGRATION_POLICY IMAGE_ARCHIVE IMAGE_MANIFEST" >&2 } case "$action" in @@ -26,6 +28,13 @@ if [[ "$root" != "/srv/who_need_help-production" ]]; then exit 2 fi +for command in curl docker git gzip jq pg_restore sha256sum; do + command -v "$command" >/dev/null 2>&1 || { + echo "Required production command is unavailable: $command" >&2 + exit 2 + } +done + env_file="$root/.env" if [[ ! -f "$env_file" || "$(stat -c '%a' "$env_file")" != 600 ]]; then echo "Production .env is missing or does not have mode 0600." >&2 @@ -122,7 +131,8 @@ if [[ "$action" == "plan" ]]; then fi if [[ -z "$bundle" || -z "$target_commit" || -z "$backup" || - -z "$expected_migration_policy" ]]; then + -z "$expected_migration_policy" || -z "$image_archive" || + -z "$image_manifest" ]]; then usage exit 2 fi @@ -133,7 +143,9 @@ if [[ "${WNH_PRODUCTION_RELEASE_CONFIRM:-}" != "$expected_confirmation" ]]; then exit 2 fi -for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" "$backup.metadata"; do +for required_file in "$bundle" "$bundle.sha256" "$backup" "$backup.sha256" \ + "$backup.metadata" "$image_archive" "$image_archive.sha256" \ + "$image_manifest"; do [[ -f "$required_file" ]] || { echo "Required release evidence is missing: $required_file" >&2 exit 2 @@ -149,6 +161,14 @@ done sha256sum --check "$(basename -- "$backup.sha256")" >/dev/null ) pg_restore --list "$backup" >/dev/null +( + cd "$(dirname -- "$image_archive")" + sha256sum --check "$(basename -- "$image_archive.sha256")" >/dev/null +) +gzip -t "$image_archive" +grep -Fx 'format=1' "$image_manifest" >/dev/null +grep -Fx "commit=$target_commit" "$image_manifest" >/dev/null +grep -Fx 'platform=linux/amd64' "$image_manifest" >/dev/null git -C "$root" bundle verify "$bundle" >/dev/null bundle_head=$(git -C "$root" bundle list-heads "$bundle" | awk '$2 == "HEAD" {print $1}') @@ -277,14 +297,8 @@ rollback_runtime() { exit "$status" } case "$app_topology" in - compact) - build_services=(migrate) - runtime_services=(app) - ;; - split) - build_services=(docker-api-proxy proxy migrate) - runtime_services=(docker-api-proxy proxy web worker) - ;; + compact) runtime_services=(app) ;; + split) runtime_services=(docker-api-proxy proxy web worker) ;; esac verify_candidate_runtime() { @@ -344,7 +358,42 @@ revision_changed=true "$root/scripts/check-environment-readiness.sh" \ "$env_file" --require-server-release -"$root/scripts/compose.sh" "$env_file" build "${build_services[@]}" +expected_images=("$(read_value APP_IMAGE)") +if [[ "$app_topology" == split ]]; then + expected_images+=( + "$(read_value SOCKET_PROXY_IMAGE)" + "$("$root/scripts/compose.sh" "$env_file" config --format json | jq -er '.services.proxy.image')" + ) +fi + +grep -Fx "topology=$app_topology" "$image_manifest" >/dev/null +test "$(grep -c '^image=' "$image_manifest")" = "${#expected_images[@]}" +for image in "${expected_images[@]}"; do + awk -F'|' -v image="$image" ' + $1 == "image=" image && $2 ~ /^sha256:[0-9a-f]+$/ { found = 1 } + END { if (!found) exit 1 } + ' "$image_manifest" +done + +gzip -dc "$image_archive" | docker load >/dev/null +for image in "${expected_images[@]}"; do + expected_id=$( + awk -F'|' -v image="$image" '$1 == "image=" image {print $2}' \ + "$image_manifest" + ) + [[ -n "$expected_id" ]] || { + echo "Image manifest is missing the expected image ID: $image" >&2 + exit 2 + } + [[ "$(docker image inspect --format '{{.Id}}' "$image")" == "$expected_id" ]] || { + echo "Loaded image ID does not match the signed manifest: $image" >&2 + exit 2 + } + [[ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" == linux/amd64 ]] || { + echo "Loaded production image is not linux/amd64: $image" >&2 + exit 2 + } +done if [[ "$migration_policy" == "forward_only" ]]; then echo "Stopping the old application before the forward-only migration boundary." diff --git a/scripts/production-release.sh b/scripts/production-release.sh index 7c6a634..fce211f 100755 --- a/scripts/production-release.sh +++ b/scripts/production-release.sh @@ -15,7 +15,7 @@ case "$action" in ;; esac -for command in git pg_restore scp sha256sum ssh; do +for command in docker git gzip mktemp pg_restore scp sha256sum ssh; do command -v "$command" >/dev/null 2>&1 || { echo "Required command is unavailable: $command" >&2 exit 2 @@ -127,14 +127,34 @@ fi "$ROOT/scripts/prepare-production-release.sh" release_dir="$ROOT/output/releases/$local_commit" bundle="$release_dir/who_need_help-$local_commit.bundle" +image_archive="$release_dir/who_need_help-$local_commit-images-linux-amd64.tar.gz" +image_checksum="$image_archive.sha256" +image_manifest="$release_dir/who_need_help-$local_commit-images.manifest" + +production_env=$(mktemp) +cleanup_production_env() { + trap - EXIT HUP INT TERM + rm -f "$production_env" +} +trap cleanup_production_env EXIT HUP INT TERM +scp -p "$ssh_target:$remote_root/.env" "$production_env" +chmod 600 "$production_env" +"$ROOT/scripts/prepare-production-images.sh" "$production_env" +cleanup_production_env + remote_release_dir="$remote_root/output/releases/incoming" remote_bundle="$remote_release_dir/$(basename -- "$bundle")" +remote_image_archive="$remote_release_dir/$(basename -- "$image_archive")" +remote_image_manifest="$remote_release_dir/$(basename -- "$image_manifest")" timestamp=$(date -u +%Y%m%dT%H%M%SZ) remote_backup="$remote_root/output/backups/production/pre-$timestamp-${local_commit:0:12}.dump" ssh -o BatchMode=yes "$ssh_target" \ "install -d -m 700 '$remote_release_dir'" -scp -p "$bundle" "$bundle.sha256" "$ssh_target:$remote_release_dir/" +scp -p \ + "$bundle" "$bundle.sha256" \ + "$image_archive" "$image_checksum" "$image_manifest" \ + "$ssh_target:$remote_release_dir/" ssh -o BatchMode=yes "$ssh_target" \ "bash -s -- '$remote_root/.env' '$remote_backup' production" \ @@ -160,7 +180,7 @@ quoted_forward_confirmation=$( printf '%q' "${WNH_PRODUCTION_FORWARD_ONLY_CONFIRM:-}" ) ssh -o BatchMode=yes "$ssh_target" \ - "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy'" \ + "WNH_PRODUCTION_RELEASE_CONFIRM=$quoted_confirmation WNH_PRODUCTION_FORWARD_ONLY_CONFIRM=$quoted_forward_confirmation bash -s -- apply '$remote_root' '$expected_domain' '$remote_bundle' '$local_commit' '$remote_backup' '$migration_policy' '$remote_image_archive' '$remote_image_manifest'" \ <"$ROOT/scripts/production-release-remote.sh" echo "Production release and public health verification completed."