From ea3c527b2405c340e297765e4e37694e28908e1d Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Tue, 21 Jul 2026 17:01:15 +0300 Subject: [PATCH] Add reproducible public staging environment --- scripts/init-staging-env.sh | 194 ++++++++++++++++++++++++++++++++ scripts/quality.sh | 36 ++++++ scripts/staging-full-e2e-run.sh | 18 ++- 3 files changed, 244 insertions(+), 4 deletions(-) create mode 100755 scripts/init-staging-env.sh diff --git a/scripts/init-staging-env.sh b/scripts/init-staging-env.sh new file mode 100755 index 0000000..7e83edf --- /dev/null +++ b/scripts/init-staging-env.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +domain=${1:-} +target=${2:-"$ROOT/.env.staging"} + +if [[ -z "$domain" ]]; then + echo "Usage: $0 DOMAIN [OUTPUT_FILE]" >&2 + exit 1 +fi + +if [[ ! "$domain" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$ ]]; then + echo "DOMAIN must be a lowercase ASCII DNS hostname without a scheme, port, or path." >&2 + exit 1 +fi + +if [[ "$target" != /* ]]; then + target="$ROOT/$target" +fi + +for command in awk docker mktemp openssl stat; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "Required command is unavailable: $command" >&2 + exit 1 + fi +done + +if [[ ! -S /var/run/docker.sock ]]; then + echo "/var/run/docker.sock is unavailable; run this on the target Docker host." >&2 + exit 1 +fi + +if [[ -e "$target" ]]; then + echo "Refusing to overwrite existing staging environment: $target" >&2 + exit 1 +fi + +target_dir=$(dirname -- "$target") +if [[ ! -d "$target_dir" ]]; then + echo "Output directory does not exist: $target_dir" >&2 + exit 1 +fi + +compose_project_name=${STAGING_COMPOSE_PROJECT_NAME:-who_need_help_staging} +public_edge_network=${STAGING_PUBLIC_EDGE_NETWORK:-who_need_help_public_edge} +public_upstream_name=${STAGING_PUBLIC_UPSTREAM_NAME:-who-need-help-staging} +http_bind_address=${STAGING_HTTP_BIND_ADDRESS:-127.0.0.1} +http_port=${STAGING_HTTP_PORT:-4011} +mailpit_bind_address=${STAGING_MAILPIT_BIND_ADDRESS:-127.0.0.1} +mailpit_port=${STAGING_MAILPIT_PORT:-8027} +codex_session_id=${STAGING_CODEX_SESSION_ID:-} +google_oauth_client_id=${STAGING_GOOGLE_OAUTH_CLIENT_ID:-} +google_oauth_client_secret=${STAGING_GOOGLE_OAUTH_CLIENT_SECRET:-} +support_inbox_address=${STAGING_SUPPORT_INBOX_ADDRESS:-} + +[[ "$compose_project_name" =~ ^[a-zA-Z0-9_-]+$ ]] || { + echo "STAGING_COMPOSE_PROJECT_NAME contains unsupported characters." >&2 + exit 1 +} +[[ "$public_edge_network" =~ ^[a-zA-Z0-9_-]+$ ]] || { + echo "STAGING_PUBLIC_EDGE_NETWORK contains unsupported characters." >&2 + exit 1 +} +[[ "$public_upstream_name" =~ ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "STAGING_PUBLIC_UPSTREAM_NAME must be a lowercase Docker DNS alias." >&2 + exit 1 +} +[[ -n "$codex_session_id" ]] || { + echo "STAGING_CODEX_SESSION_ID is required for the Build Week feedback page." >&2 + exit 1 +} +if [[ -n "$google_oauth_client_id" || -n "$google_oauth_client_secret" ]]; then + [[ -n "$google_oauth_client_id" && -n "$google_oauth_client_secret" ]] || { + echo "Staging Google OAuth client ID and secret must either both be set or both be empty." >&2 + exit 1 + } +fi + +for pair in "HTTP:$http_port" "MAILPIT:$mailpit_port"; do + value=${pair#*:} + if [[ ! "$value" =~ ^[0-9]+$ ]] || ((value < 1 || value > 65535)); then + echo "${pair%%:*} port must be between 1 and 65535." >&2 + exit 1 + fi +done +[[ "$http_port" != "$mailpit_port" ]] || { + echo "Staging HTTP and Mailpit ports must be different." >&2 + exit 1 +} + +postgres_password=$(openssl rand -hex 32) +secret_key_base=$(openssl rand -hex 64) +handover_secret=$(openssl rand -hex 64) +release_cookie=$(openssl rand -hex 64) +metrics_token=$(openssl rand -hex 32) +docker_socket_gid=$(stat -c '%g' /var/run/docker.sock) +postgres_db=who_need_help_staging_e2e +database_url="ecto://postgres:$postgres_password@db/$postgres_db" + +tmp=$(mktemp "$target_dir/.staging-env.XXXXXX") +trap 'rm -f "$tmp"' EXIT HUP INT TERM +chmod 600 "$tmp" + +DOMAIN=$domain \ +COMPOSE_PROJECT_NAME_VALUE=$compose_project_name \ +PUBLIC_EDGE_NETWORK_VALUE=$public_edge_network \ +PUBLIC_UPSTREAM_NAME_VALUE=$public_upstream_name \ +HTTP_BIND_ADDRESS_VALUE=$http_bind_address \ +HTTP_PORT_VALUE=$http_port \ +MAILPIT_BIND_ADDRESS_VALUE=$mailpit_bind_address \ +MAILPIT_PORT_VALUE=$mailpit_port \ +DOCKER_SOCKET_GID_VALUE=$docker_socket_gid \ +POSTGRES_DB_VALUE=$postgres_db \ +POSTGRES_PASSWORD_VALUE=$postgres_password \ +DATABASE_URL_VALUE=$database_url \ +SECRET_KEY_BASE_VALUE=$secret_key_base \ +HANDOVER_SECRET_VALUE=$handover_secret \ +RELEASE_COOKIE_VALUE=$release_cookie \ +METRICS_TOKEN_VALUE=$metrics_token \ +GOOGLE_OAUTH_CLIENT_ID_VALUE=$google_oauth_client_id \ +GOOGLE_OAUTH_CLIENT_SECRET_VALUE=$google_oauth_client_secret \ +SUPPORT_INBOX_ADDRESS_VALUE=$support_inbox_address \ +CODEX_SESSION_ID_VALUE=$codex_session_id \ + awk ' + BEGIN { + replacement["DEPLOYMENT_TARGET"] = "compose" + replacement["DEPLOYMENT_ENV"] = "development" + replacement["COMPOSE_PROJECT_NAME"] = ENVIRON["COMPOSE_PROJECT_NAME_VALUE"] + replacement["APP_TOPOLOGY"] = "compact" + replacement["DATABASE_MODE"] = "container" + replacement["HTTP_BIND_ADDRESS"] = ENVIRON["HTTP_BIND_ADDRESS_VALUE"] + replacement["HTTP_PORT"] = ENVIRON["HTTP_PORT_VALUE"] + replacement["PUBLIC_EDGE_ENABLED"] = "true" + replacement["PUBLIC_EDGE_NETWORK"] = ENVIRON["PUBLIC_EDGE_NETWORK_VALUE"] + replacement["PUBLIC_UPSTREAM_NAME"] = ENVIRON["PUBLIC_UPSTREAM_NAME_VALUE"] + replacement["MAILPIT_BIND_ADDRESS"] = ENVIRON["MAILPIT_BIND_ADDRESS_VALUE"] + replacement["MAILPIT_PORT"] = ENVIRON["MAILPIT_PORT_VALUE"] + replacement["DOCKER_SOCKET_GID"] = ENVIRON["DOCKER_SOCKET_GID_VALUE"] + replacement["TRAEFIK_TRUSTED_IPS"] = "127.0.0.1/32" + replacement["TRAEFIK_PROJECT_CONSTRAINT"] = ENVIRON["COMPOSE_PROJECT_NAME_VALUE"] + replacement["TRAEFIK_APP_NAME"] = "who-need-help-staging" + replacement["TRAEFIK_DOCKER_NETWORK"] = ENVIRON["COMPOSE_PROJECT_NAME_VALUE"] "_ingress" + replacement["PHX_HOST"] = ENVIRON["DOMAIN"] + replacement["PHX_SCHEME"] = "https" + replacement["PHX_URL_PORT"] = "443" + replacement["WNH_DEBUG_BASE_URL"] = "https://" ENVIRON["DOMAIN"] + replacement["WNH_BASE_URL"] = "https://" ENVIRON["DOMAIN"] + replacement["POSTGRES_DB"] = ENVIRON["POSTGRES_DB_VALUE"] + replacement["POSTGRES_USER"] = "postgres" + replacement["POSTGRES_PASSWORD"] = ENVIRON["POSTGRES_PASSWORD_VALUE"] + replacement["DATABASE_URL"] = ENVIRON["DATABASE_URL_VALUE"] + replacement["DATABASE_SOCKET_DIR"] = "" + replacement["SECRET_KEY_BASE"] = ENVIRON["SECRET_KEY_BASE_VALUE"] + replacement["HANDOVER_SECRET"] = ENVIRON["HANDOVER_SECRET_VALUE"] + replacement["RELEASE_COOKIE"] = ENVIRON["RELEASE_COOKIE_VALUE"] + replacement["METRICS_TOKEN"] = ENVIRON["METRICS_TOKEN_VALUE"] + replacement["EMAIL_DELIVERY_PROVIDER"] = "smtp" + replacement["SMTP_RELAY"] = "mailpit" + replacement["SMTP_PORT"] = "1025" + replacement["SMTP_USERNAME"] = "" + replacement["SMTP_PASSWORD"] = "" + replacement["SMTP_AUTH"] = "never" + replacement["SMTP_TLS"] = "never" + replacement["SMTP_SSL"] = "false" + replacement["EMAIL_FROM_NAME"] = "Who Need Help Staging" + replacement["EMAIL_FROM_ADDRESS"] = "staging@" ENVIRON["DOMAIN"] + replacement["SUPPORT_INBOX_ADDRESS"] = ENVIRON["SUPPORT_INBOX_ADDRESS_VALUE"] + replacement["GOOGLE_OAUTH_CLIENT_ID"] = ENVIRON["GOOGLE_OAUTH_CLIENT_ID_VALUE"] + replacement["GOOGLE_OAUTH_CLIENT_SECRET"] = ENVIRON["GOOGLE_OAUTH_CLIENT_SECRET_VALUE"] + replacement["CODEX_SESSION_ID"] = ENVIRON["CODEX_SESSION_ID_VALUE"] + } + { + separator = index($0, "=") + key = separator > 1 ? substr($0, 1, separator - 1) : "" + if (key in replacement) { + print key "=" replacement[key] + } else { + print + } + } + ' "$ROOT/.env.example" >"$tmp" + +mv "$tmp" "$target" +chmod 600 "$target" +trap - EXIT HUP INT TERM + +unset postgres_password secret_key_base handover_secret release_cookie metrics_token +unset google_oauth_client_secret + +"$ROOT/scripts/compose.sh" "$target" config --quiet +echo "Generated independent staging secrets without printing them." +echo "Created mode-0600 staging environment: $target" diff --git a/scripts/quality.sh b/scripts/quality.sh index 6c61b13..fe9b971 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -100,6 +100,42 @@ if ./scripts/init-edge-env.sh help.test staging.help.test "$edge_env" >/dev/null echo "Edge environment initializer overwrote an existing file." >&2 exit 1 fi +staging_env="$scan_dir/.env.staging" +if ./scripts/init-staging-env.sh staging.help.test \ + "$scan_dir/.env.staging.missing-codex" >/dev/null 2>&1; then + echo "Staging environment initializer accepted a missing Codex session ID." >&2 + exit 1 +fi +STAGING_CODEX_SESSION_ID=00000000-0000-0000-0000-000000000001 \ +STAGING_GOOGLE_OAUTH_CLIENT_ID=quality-staging-client \ +STAGING_GOOGLE_OAUTH_CLIENT_SECRET=quality-staging-secret \ + ./scripts/init-staging-env.sh staging.help.test "$staging_env" >/dev/null +test "$(stat -c '%a' "$staging_env")" = 600 +grep -Fx 'DEPLOYMENT_ENV=development' "$staging_env" >/dev/null +grep -Fx 'COMPOSE_PROJECT_NAME=who_need_help_staging' "$staging_env" >/dev/null +grep -Fx 'APP_TOPOLOGY=compact' "$staging_env" >/dev/null +grep -Fx 'DATABASE_MODE=container' "$staging_env" >/dev/null +grep -Fx 'PUBLIC_EDGE_ENABLED=true' "$staging_env" >/dev/null +grep -Fx 'PUBLIC_UPSTREAM_NAME=who-need-help-staging' "$staging_env" >/dev/null +grep -Fx 'PHX_HOST=staging.help.test' "$staging_env" >/dev/null +grep -Fx 'PHX_SCHEME=https' "$staging_env" >/dev/null +grep -Fx 'PHX_URL_PORT=443' "$staging_env" >/dev/null +grep -Fx 'EMAIL_DELIVERY_PROVIDER=smtp' "$staging_env" >/dev/null +grep -Fx 'SMTP_RELAY=mailpit' "$staging_env" >/dev/null +grep -Fx 'GOOGLE_OAUTH_CLIENT_ID=quality-staging-client' "$staging_env" >/dev/null +"$ROOT/scripts/compose.sh" "$staging_env" config --quiet +if STAGING_CODEX_SESSION_ID=00000000-0000-0000-0000-000000000001 \ + STAGING_GOOGLE_OAUTH_CLIENT_ID=quality-staging-client \ + ./scripts/init-staging-env.sh staging.help.test \ + "$scan_dir/.env.staging.partial-google" >/dev/null 2>&1; then + echo "Staging environment initializer accepted partial Google OAuth credentials." >&2 + exit 1 +fi +if STAGING_CODEX_SESSION_ID=00000000-0000-0000-0000-000000000001 \ + ./scripts/init-staging-env.sh staging.help.test "$staging_env" >/dev/null 2>&1; then + echo "Staging environment initializer overwrote an existing file." >&2 + exit 1 +fi production_env="$scan_dir/.env.production" missing_codex_env="$scan_dir/.env.production.missing-codex" if PRODUCTION_TRAEFIK_TRUSTED_IPS=172.20.0.1/32 \ diff --git a/scripts/staging-full-e2e-run.sh b/scripts/staging-full-e2e-run.sh index 0c6f506..7c51481 100755 --- a/scripts/staging-full-e2e-run.sh +++ b/scripts/staging-full-e2e-run.sh @@ -5,10 +5,14 @@ umask 077 ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) cd "$ROOT" -ENV_FILE="$ROOT/.env" +ENV_FILE=${1:-"$ROOT/.env"} + +if [[ "$ENV_FILE" != /* ]]; then + ENV_FILE="$ROOT/$ENV_FILE" +fi if [[ ! -f "$ENV_FILE" ]]; then - echo "Missing $ENV_FILE." >&2 + echo "Missing staging environment: $ENV_FILE." >&2 exit 1 fi @@ -48,7 +52,11 @@ chmod 700 "$ROOT/output" "$ROOT/output/staging-full-e2e" "$output_dir" fixture_password=$(openssl rand -base64 36 | tr -d '\n') prepared=0 -db_container=$(docker compose ps -q db) +compose() { + "$ROOT/scripts/compose.sh" "$ENV_FILE" "$@" +} + +db_container=$(compose ps -q db) if [[ -z "$db_container" ]]; then echo "The ordinary Compose database container is not running." >&2 @@ -69,7 +77,9 @@ fi snapshot_database() { local destination=$1 - docker compose exec -T db sh -c \ + # Expand database variables inside the database container, not on the host. + # shellcheck disable=SC2016 + compose exec -T db sh -c \ 'psql --no-psqlrc --set ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \ >"$destination" <<'SQL' BEGIN READ ONLY;