who_need_help/scripts/production-full-e2e.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

# Production browser verification is intentionally opt-in and run-scoped.

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
MODE=${1:-plan}
RUN_ID=${2:-"production-e2e-$(date -u +%Y%m%d%H%M%S)"}
SSH_TARGET=${WNH_PRODUCTION_E2E_SSH_TARGET:-whoneedhelp}
REMOTE_ROOT=${WNH_PRODUCTION_E2E_REMOTE_ROOT:-/srv/who_need_help-production}
BASE_URL=https://whoneedhelp.com

usage() {
  cat <<'EOF'
Usage:
  ./scripts/production-full-e2e.sh plan [run-id]
  WNH_PRODUCTION_E2E_CONFIRM='<exact value printed by plan>' \
    ./scripts/production-full-e2e.sh run [run-id]

The run creates only run-scoped synthetic users and records, exercises the
two-user help and moderated activity browser flows, and removes the exact
fixture on success, failure, or interrupt. It never resets the database.
EOF
}

case "$MODE" in
  plan | run) ;;
  *) usage >&2; exit 1 ;;
esac

if [[ ! "$RUN_ID" =~ ^[a-z0-9-]+$ ]]; then
  echo "run-id may contain only lowercase letters, numbers, and dashes." >&2
  exit 1
fi

for command in curl docker git jq mktemp openssl sha256sum ssh tar; do
  command -v "$command" >/dev/null 2>&1 || {
    echo "Required command is unavailable: $command" >&2
    exit 2
  }
done

inventory=$(
  ssh -o BatchMode=yes "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$RUN_ID" <<'REMOTE'
set -euo pipefail
root=$1
run_id=$2
env_file="$root/.env"

if [[ ! -f "$env_file" ]]; then
  echo "Missing production environment: $env_file" >&2
  exit 1
fi

python3 - "$env_file" <<'PY'
import shlex
import sys

path = sys.argv[1]
wanted = {
    "COMPOSE_PROJECT_NAME",
    "DATABASE_MODE",
    "DEPLOYMENT_ENV",
    "PHX_HOST",
    "POSTGRES_DB",
    "WNH_BASE_URL",
}
values = {}
with open(path, encoding="utf-8") as handle:
    for raw_line in handle:
        line = raw_line.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        key, value = line.split("=", 1)
        if key not in wanted:
            continue
        parsed = shlex.split(value, comments=False, posix=True)
        values[key] = parsed[0] if parsed else ""

missing = sorted(key for key in wanted if not values.get(key))
if missing:
    raise SystemExit("Missing production identity settings: " + ", ".join(missing))

for key in sorted(wanted):
    print(f"{key.lower()}={values[key]}")
PY

commit=$(git -C "$root" rev-parse HEAD)
project=$(
  python3 - "$env_file" <<'PY'
import shlex
import sys

with open(sys.argv[1], encoding="utf-8") as handle:
    for raw_line in handle:
        line = raw_line.strip()
        if line.startswith("COMPOSE_PROJECT_NAME="):
            parsed = shlex.split(line.split("=", 1)[1], comments=False, posix=True)
            if parsed:
                print(parsed[0])
            break
PY
)

if [[ -z "$project" ]]; then
  echo "Missing normalized COMPOSE_PROJECT_NAME." >&2
  exit 1
fi
mapfile -t containers < <(
  docker ps \
    --filter "label=com.docker.compose.project=$project" \
    --filter "label=com.docker.compose.service=app" \
    --format '{{.Names}}'
)

if [[ "${#containers[@]}" -ne 1 ]]; then
  echo "Expected exactly one compact production app container; observed ${#containers[@]}." >&2
  exit 1
fi

container=${containers[0]}
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
restart_count=$(docker inspect --format '{{.RestartCount}}' "$container")
prefix="wnh-staging-e2e-$run_id-%"
prefix_count=$(
  docker exec "$container" /app/bin/who_need_help rpc \
    "result = WhoNeedHelp.Repo.query!(\"SELECT count(*) FROM users WHERE email LIKE \\\$1\", [\"$prefix\"], log: false); IO.puts(result.rows |> hd() |> hd())" |
    tail -n 1
)

printf 'commit=%s\n' "$commit"
printf 'container=%s\n' "$container"
printf 'health=%s\n' "$health"
printf 'restart_count=%s\n' "$restart_count"
printf 'fixture_prefix_count=%s\n' "$prefix_count"
REMOTE
)

value() {
  local name=$1
  printf '%s\n' "$inventory" | sed -n "s/^${name}=//p" | tail -n 1
}

commit=$(value commit)
container=$(value container)
database=$(value postgres_db)
project=$(value compose_project_name)

if [[ "$(value deployment_env)" != production ]] ||
  [[ "$(value phx_host)" != whoneedhelp.com ]] ||
  [[ "$(value wnh_base_url)" != "$BASE_URL" ]] ||
  [[ "$(value database_mode)" != external ]] ||
  [[ "$project" != who_need_help_production ]] ||
  [[ "$(value health)" != healthy ]] ||
  [[ "$(value fixture_prefix_count)" != 0 ]] ||
  [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]] ||
  [[ -z "$database" ]] ||
  [[ -z "$container" ]]; then
  echo "The observed target does not match the exact compact production identity." >&2
  printf '%s\n' "$inventory" >&2
  exit 1
fi

git cat-file -e "$commit^{commit}" 2>/dev/null || {
  echo "Production commit $commit is not available in the local repository." >&2
  exit 1
}

# The deployed application is always archived from its exact remote commit.
# Local development may legitimately be ahead of production; those files never
# enter either runner image. Only these explicit verifier overlays are copied
# below, and their hashes are retained with the evidence bundle.
for verifier_path in \
  lib/mix/tasks/wnh.staging_full_e2e.ex \
  e2e/tests/activity-moderation.spec.ts \
  e2e/tests/helpers.ts; do
  if [[ ! -f "$ROOT/$verifier_path" ]]; then
    echo "Production E2E verifier is unavailable: $verifier_path" >&2
    exit 1
  fi
done

confirmation="whoneedhelp.com:${commit}:${database}:${RUN_ID}:full-browser-e2e"

cat <<EOF
Verified production target:
  URL: $BASE_URL
  SSH target: $SSH_TARGET
  checkout: $REMOTE_ROOT
  Compose project: $project
  database: $database
  commit: $commit
  app container: $container
  restart count: $(value restart_count)
  existing run-scoped fixture users: 0

Exact temporary mutation scope:
  - six confirmed synthetic users under wnh-staging-e2e-$RUN_ID-*;
  - their help requests, assignments, chats, positions, handover, reviews;
  - their activity, participation, group chat, report and category proposal;
  - their notifications, audit events and associated Oban jobs;
  - no database reset, migration, real-user role/status change, email delivery,
    Caddy change, test-project change, payment, iOS, or KYC action.

The exact manifest-owned fixture is removed on success, failure, or interrupt.
Cleanup refuses cross-fixture relationships and verifies that the run prefix is absent.

To execute exactly this plan, set:
  WNH_PRODUCTION_E2E_CONFIRM='$confirmation'
EOF

if [[ "$MODE" == plan ]]; then
  exit 0
fi

if [[ "${WNH_PRODUCTION_E2E_CONFIRM:-}" != "$confirmation" ]]; then
  echo "Run refused: WNH_PRODUCTION_E2E_CONFIRM does not match this verified plan." >&2
  exit 1
fi

output_dir="$ROOT/output/production-full-e2e/$RUN_ID"
remote_output="$REMOTE_ROOT/output/production-full-e2e/$RUN_ID"
short=${commit:0:12}
tools_image="who-need-help:production-e2e-tools-$short"
browser_image="who-need-help-e2e-tests:production-$short"
archive_dir=$(mktemp -d "$ROOT/tmp/production-e2e-source.XXXXXX")
fixture_password=$(openssl rand -base64 36 | tr -d '\n')
prepared=0

if [[ -e "$output_dir" ]]; then
  echo "Local evidence directory already exists: $output_dir" >&2
  exit 1
fi

mkdir -p "$output_dir/browser"
chmod 700 "$ROOT/output" "$ROOT/output/production-full-e2e" "$output_dir" "$output_dir/browser"
printf '%s\n' "$inventory" >"$output_dir/environment.txt"
printf '%s\n' "$confirmation" >"$output_dir/confirmation.txt"

snapshot() {
  local destination=$1
  ssh -o BatchMode=yes "$SSH_TARGET" bash -s -- "$container" "$RUN_ID" <<'REMOTE' >"$destination"
set -euo pipefail
container=$1
run_id=$2
docker exec "$container" /app/bin/who_need_help rpc '
  alias WhoNeedHelp.Repo
  prefix = "wnh-staging-e2e-'"$run_id"'-%"
  tables = ~w(users help_requests help_assignments messages tracking_sessions tracking_positions reviews activities activity_participants activity_messages reports category_proposals category_votes audit_events notifications oban_jobs)
  counts =
    Map.new(tables, fn table ->
      result = Repo.query!("SELECT count(*) FROM #{table}", [], log: false)
      {table, result.rows |> hd() |> hd()}
    end)
  prefix_count = Repo.query!("SELECT count(*) FROM users WHERE email LIKE $1", [prefix], log: false).rows |> hd() |> hd()
  IO.puts(Jason.encode!(%{captured_at: DateTime.utc_now(), counts: counts, fixture_prefix_count: prefix_count}))
' | tail -n 1
REMOTE
}

run_fixture() {
  local action=$1
  ssh -o BatchMode=yes "$SSH_TARGET" bash -s -- \
    "$REMOTE_ROOT" "$remote_output" "$tools_image" "$database" "$RUN_ID" "$fixture_password" "$action" <<'REMOTE'
set -euo pipefail
root=$1
output=$2
image=$3
database=$4
run_id=$5
password=$6
action=$7
mkdir -p "$output"
chmod 700 "$output"
docker run --rm \
  --network host \
  --env-file "$root/.env" \
  --env APP_ROLE=migrate \
  --env "WNH_STAGING_E2E_EXPECTED_DATABASE=$database" \
  --env WNH_STAGING_E2E_CONFIRM=public-staging-full-e2e \
  --env "WNH_STAGING_E2E_RUN_ID=$run_id" \
  --env "WNH_STAGING_E2E_PASSWORD=$password" \
  --env WNH_STAGING_E2E_MANIFEST_PATH=/output/fixture.json \
  --volume /var/run/postgresql:/var/run/postgresql \
  --volume "$output:/output" \
  "$image" \
  mix wnh.staging_full_e2e "$action"
REMOTE
}

cleanup() {
  local status=$?
  trap - EXIT HUP INT TERM
  local copied_manifest=0

  if [[ "$prepared" -eq 1 ]]; then
    if ! run_fixture cleanup >"$output_dir/fixture-cleanup.log" 2>&1; then
      echo "Exact production E2E fixture cleanup failed; inspect $output_dir." >&2
      status=1
    fi

    if ssh -o BatchMode=yes "$SSH_TARGET" docker run --rm \
      --volume "$remote_output:/output:ro" \
      "$tools_image" \
      cat /output/fixture.json >"$output_dir/fixture.json"; then
      copied_manifest=1
    else
      echo "Production E2E cleanup manifest could not be copied." >&2
      status=1
    fi

    if [[ "$copied_manifest" -eq 1 ]] &&
      ! jq -e '
        .cleanup_verified == true and
        (.cleanup_targets.users | length) >= 6 and
        (.cleanup_deleted_counts.users | type) == "number" and
        .cleanup_verified_at != null
      ' "$output_dir/fixture.json" >/dev/null; then
      echo "Production E2E cleanup manifest lacks exact run-scoped verification." >&2
      status=1
    fi
  fi

  snapshot "$output_dir/database-after.json" 2>"$output_dir/database-after-error.log" || status=1

  if [[ -s "$output_dir/database-after.json" ]] &&
    [[ "$(jq -r '.fixture_prefix_count' "$output_dir/database-after.json")" != 0 ]]; then
    echo "Run-scoped production fixture users remain after cleanup." >&2
    status=1
  fi

  ssh -o BatchMode=yes "$SSH_TARGET" \
    "rm -rf -- '$remote_output'; docker image rm '$tools_image' >/dev/null 2>&1 || true" || status=1
  docker image rm "$tools_image" "$browser_image" >/dev/null 2>&1 || true
  rm -rf -- "$archive_dir"
  unset fixture_password
  exit "$status"
}
trap cleanup EXIT HUP INT TERM

curl --fail --silent --show-error --max-time 10 "$BASE_URL/healthz/ready" \
  >"$output_dir/readiness-before.json"
snapshot "$output_dir/database-before.json"

git archive "$commit" | tar -x -C "$archive_dir"
# The deployed application remains the exact production commit. Only the
# run-scoped fixture task and browser assertion are overlaid into throwaway
# runner images so the verifier can evolve without deploying application code.
cp "$ROOT/lib/mix/tasks/wnh.staging_full_e2e.ex" \
  "$archive_dir/lib/mix/tasks/wnh.staging_full_e2e.ex"
cp "$ROOT/e2e/tests/activity-moderation.spec.ts" \
  "$archive_dir/e2e/tests/activity-moderation.spec.ts"
cp "$ROOT/e2e/tests/helpers.ts" "$archive_dir/e2e/tests/helpers.ts"
sha256sum \
  "$ROOT/lib/mix/tasks/wnh.staging_full_e2e.ex" \
  "$ROOT/e2e/tests/activity-moderation.spec.ts" \
  "$ROOT/e2e/tests/helpers.ts" \
  >"$output_dir/harness-sha256.txt"
# This script starts with umask 077 so evidence and credentials remain private.
# The archived source is copied into a browser image that later runs under the
# invoking host UID; make only this throwaway source tree readable first.
chmod -R u+rwX,go+rX "$archive_dir"

docker build --target load_tools --tag "$tools_image" "$archive_dir" \
  >"$output_dir/tools-build.log"
docker build --tag "$browser_image" "$archive_dir/e2e" \
  >"$output_dir/browser-build.log"

docker save "$tools_image" |
  ssh -o BatchMode=yes "$SSH_TARGET" docker load \
    >"$output_dir/tools-load.log"

run_fixture prepare >"$output_dir/fixture-prepare.log"
prepared=1

docker run --rm \
  --network host \
  --user "$(id -u):$(id -g)" \
  --env "BASE_URL=$BASE_URL" \
  --env MAILPIT_URL=http://127.0.0.1:1 \
  --env "E2E_RUN_ID=$RUN_ID" \
  --env "E2E_FIXTURE_PASSWORD=$fixture_password" \
  --env "E2E_ADMIN_EMAIL=wnh-staging-e2e-$RUN_ID-admin@example.invalid" \
  --env HOME=/tmp \
  --volume "$output_dir/browser:/work/output" \
  "$browser_image" \
  npx playwright test --project=chromium \
    tests/mutual-aid.spec.ts tests/activity-moderation.spec.ts |
  tee "$output_dir/browser-console.log"

curl --fail --silent --show-error --max-time 10 "$BASE_URL/healthz/ready" \
  >"$output_dir/readiness-after-browser.json"

echo "Production full browser E2E passed. Evidence: $output_dir"