Preserve JSON values during restore drills

scripts/quality.sh

@@ -131,6 +131,7 @@ docker run --rm \
 
 echo "Checking production restore PostgreSQL readiness"
 bash test/scripts/production_offsite_backup_readiness_test.sh
+./scripts/restore-drill-env-regression.sh
 
 echo "Checking the production read-only load safety boundary"
 ./scripts/production-readonly-load-drill.sh

scripts/restore-drill-compose.sh

@@ -22,24 +22,39 @@ if [ ! -f "$checksum" ]; then
   exit 1
 fi
 
-if [ ! -f "$ROOT/.env" ]; then
-  echo "Missing $ROOT/.env." >&2
+env_file="$ROOT/.env"
+
+if [ ! -f "$env_file" ]; then
+  echo "Missing $env_file." >&2
   exit 1
 fi
 
-set -a
-# shellcheck source=/dev/null
-. "$ROOT/.env"
-set +a
+read_env_value() {
+  key=$1
 
-if [ "${DATABASE_MODE:-container}" != container ]; then
+  awk -v key="$key" '
+    index($0, key "=") == 1 {
+      print substr($0, length(key) + 2)
+      found = 1
+      exit
+    }
+    END { if (!found) exit 1 }
+  ' "$env_file"
+}
+
+database_mode=$(read_env_value DATABASE_MODE 2>/dev/null || printf container)
+postgres_db=$(read_env_value POSTGRES_DB 2>/dev/null || true)
+postgres_user=$(read_env_value POSTGRES_USER 2>/dev/null || true)
+database_url=$(read_env_value DATABASE_URL 2>/dev/null || true)
+
+if [ "$database_mode" != container ]; then
   echo "restore-drill-compose.sh only operates on DATABASE_MODE=container." >&2
   exit 1
 fi
 
-: "${POSTGRES_DB:?Set POSTGRES_DB in .env}"
-: "${POSTGRES_USER:?Set POSTGRES_USER in .env}"
-: "${DATABASE_URL:?Set DATABASE_URL in .env}"
+: "${postgres_db:?Set POSTGRES_DB in .env}"
+: "${postgres_user:?Set POSTGRES_USER in .env}"
+: "${database_url:?Set DATABASE_URL in .env}"
 
 dump_dir=$(CDPATH='' cd -- "$(dirname -- "$dump")" && pwd)
 dump_name=$(basename -- "$dump")
@@ -49,37 +64,41 @@ dump_name=$(basename -- "$dump")
   sha256sum --check --status "$(basename -- "$checksum")"
 )
 
-docker compose exec -T db pg_restore --list <"$dump" >/dev/null
+compose() {
+  "$ROOT/scripts/compose.sh" "$env_file" "$@"
+}
+
+compose exec -T db pg_restore --list <"$dump" >/dev/null
 
 drill_db="who_need_help_restore_drill_$(date -u +%Y%m%d%H%M%S)_$$"
 created=false
 
 cleanup() {
   if [ "$created" = true ]; then
-    docker compose exec -T db \
-      dropdb --username "$POSTGRES_USER" --if-exists "$drill_db" >/dev/null
+    compose exec -T db \
+      dropdb --username "$postgres_user" --if-exists "$drill_db" >/dev/null
   fi
 }
 trap cleanup EXIT HUP INT TERM
 
-docker compose exec -T db \
+compose exec -T db \
   createdb \
-  --username "$POSTGRES_USER" \
+  --username "$postgres_user" \
   --template template0 \
   "$drill_db"
 created=true
 
-docker compose exec -T db \
+compose exec -T db \
   pg_restore \
-  --username "$POSTGRES_USER" \
+  --username "$postgres_user" \
   --dbname "$drill_db" \
   --exit-on-error \
   --no-owner \
   --no-privileges <"$dump"
 
 tables=$(
-  docker compose exec -T db \
-    psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+  compose exec -T db \
+    psql --username "$postgres_user" --dbname "$drill_db" --tuples-only --no-align \
     --command "
       SELECT tablename
       FROM pg_tables
@@ -94,8 +113,8 @@ total_rows=0
 
 for table in $tables; do
   restored_count=$(
-    docker compose exec -T db \
-      psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+    compose exec -T db \
+      psql --username "$postgres_user" --dbname "$drill_db" --tuples-only --no-align \
       --command "SELECT count(*) FROM \"$table\";"
   )
 
@@ -109,19 +128,19 @@ if [ "$table_count" -eq 0 ]; then
 fi
 
 postgis_version=$(
-  docker compose exec -T db \
-    psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+  compose exec -T db \
+    psql --username "$postgres_user" --dbname "$drill_db" --tuples-only --no-align \
     --command "SELECT PostGIS_Lib_Version();"
 )
 
-case "$DATABASE_URL" in
+case "$database_url" in
   *\?*)
-    database_base=${DATABASE_URL%%\?*}
-    database_query="?${DATABASE_URL#*\?}"
+    database_base=${database_url%%\?*}
+    database_query="?${database_url#*\?}"
     ;;
 
   *)
-    database_base=$DATABASE_URL
+    database_base=$database_url
     database_query=
     ;;
 esac
@@ -135,30 +154,30 @@ fi
 
 drill_database_url="$database_prefix/$drill_db$database_query"
 
-docker compose run --rm --no-deps \
+compose run --rm --no-deps \
   --env APP_ROLE=migrate \
   --env "DATABASE_URL=$drill_database_url" \
   migrate /app/bin/migrate
 
-docker compose run --rm --no-deps \
+compose run --rm --no-deps \
   --env APP_ROLE=migrate \
   --env "DATABASE_URL=$drill_database_url" \
   migrate /app/bin/who_need_help eval 'WhoNeedHelp.Release.await_migrations()'
 
 migration_count=$(
-  docker compose exec -T db \
-    psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+  compose exec -T db \
+    psql --username "$postgres_user" --dbname "$drill_db" --tuples-only --no-align \
     --command "SELECT count(*) FROM schema_migrations;"
 )
 
-docker compose exec -T db \
-  dropdb --username "$POSTGRES_USER" "$drill_db"
+compose exec -T db \
+  dropdb --username "$postgres_user" "$drill_db"
 created=false
 trap - EXIT HUP INT TERM
 
 remaining=$(
-  docker compose exec -T db \
-    psql --username "$POSTGRES_USER" --dbname postgres --tuples-only --no-align \
+  compose exec -T db \
+    psql --username "$postgres_user" --dbname postgres --tuples-only --no-align \
     --command "SELECT count(*) FROM pg_database WHERE datname = '$drill_db';"
 )
 

scripts/restore-drill-env-regression.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+source_script=${WNH_RESTORE_DRILL_SCRIPT:-"$ROOT/scripts/restore-drill-compose.sh"}
+run_dir=$(mktemp -d "$ROOT/output/restore-drill-env-regression.XXXXXX")
+fixture="$run_dir/project"
+mock_bin="$run_dir/mock-bin"
+
+cleanup() {
+  status=$?
+  trap - EXIT HUP INT TERM
+  find "$run_dir" -xdev -depth -delete 2>/dev/null || true
+  exit "$status"
+}
+trap cleanup EXIT HUP INT TERM
+
+install -d -m 700 "$fixture/scripts" "$mock_bin"
+install -m 755 "$source_script" "$fixture/scripts/restore-drill-compose.sh"
+install -m 755 "$ROOT/scripts/compose.sh" "$fixture/scripts/compose.sh"
+
+install -m 600 /dev/null "$fixture/.env"
+printf '%s\n' \
+  'DEPLOYMENT_TARGET=compose' \
+  'DEPLOYMENT_ENV=test' \
+  'COMPOSE_PROJECT_NAME=who_need_help_restore_drill_env_regression' \
+  'DATABASE_MODE=container' \
+  'APP_TOPOLOGY=compact' \
+  'PUBLIC_EDGE_ENABLED=false' \
+  'EMAIL_DELIVERY_PROVIDER=smtp' \
+  'SMTP_RELAY=smtp.example.invalid' \
+  'POSTGRES_DB=who_need_help_test' \
+  'POSTGRES_USER=who_need_help_test' \
+  'POSTGRES_PASSWORD=restore-drill-regression' \
+  'DATABASE_URL=ecto://who_need_help_test:restore-drill-regression@db/who_need_help_test' \
+  'RATE_LIMIT_POLICIES_JSON={"registration_email":{"limit":4,"window_seconds":3600}}' \
+  >"$fixture/.env"
+
+dump="$run_dir/test.dump"
+printf 'restore drill regression fixture\n' >"$dump"
+printf '%s  %s\n' "$(sha256sum "$dump" | awk '{print $1}')" \
+  "$(basename -- "$dump")" >"$dump.sha256"
+
+cat >"$mock_bin/docker" <<'EOF'
+#!/bin/sh
+set -eu
+
+if [ "${RATE_LIMIT_POLICIES_JSON+x}" = x ]; then
+  echo "RATE_LIMIT_POLICIES_JSON leaked through shell evaluation of .env." >&2
+  exit 97
+fi
+
+case "$*" in
+  *"SELECT tablename"*) printf 'example_table\n' ;;
+  *'SELECT count(*) FROM "example_table";'*) printf '1\n' ;;
+  *'SELECT PostGIS_Lib_Version();'*) printf '3.6.4\n' ;;
+  *'SELECT count(*) FROM schema_migrations;'*) printf '29\n' ;;
+  *'SELECT count(*) FROM pg_database'*) printf '0\n' ;;
+esac
+EOF
+chmod 755 "$mock_bin/docker"
+
+PATH="$mock_bin:$PATH" \
+  "$fixture/scripts/restore-drill-compose.sh" "$dump" >/dev/null
+
+echo "Restore drill treats .env as data and preserves JSON values for Compose."