Support remote Android cross-client verification

README.md

@@ -477,11 +477,18 @@ removal, and exact database cleanup:
 ./scripts/android-staging-build.sh
 ./scripts/android-staging-smoke.sh
 ./scripts/android-browser-staging-e2e.sh
+./scripts/android-browser-staging-e2e.sh --remote \
+  SSH_HOST:/absolute/path/to/test/checkout
 ```
 
 The cross-client script refuses an unexpected database, uses a unique fixture
 prefix and manifest, and compares counts across 19 application tables before
-and after cleanup. It does not delete unrelated records.
+and after cleanup. Remote mode accepts only a container-backed deployment with
+`DEPLOYMENT_ENV=test` and a Compose project ending in `_test`; it keeps the
+Android emulator and browser local, reaches the remote Mailpit API through a
+run-scoped loopback SSH tunnel, and executes fixture preparation, verification,
+and exact cleanup next to the remote test database. It does not delete
+unrelated records.
 
 ## First administrator
 

scripts/android-browser-staging-e2e.sh

@@ -8,6 +8,12 @@ cd "$ROOT"
 ENV_FILE="$ROOT/.env"
 APK="$ROOT/android/dist-staging/who-need-help-staging.apk"
 TEST_APK="$ROOT/android/dist-staging/who-need-help-staging-androidTest.apk"
+remote_target=
+remote_host=
+remote_root=
+remote_output_dir=
+remote_tools_image=
+ssh_tunnel_pid=
 run_id="$(date -u +%Y%m%d%H%M%S)-$$"
 output_dir="$ROOT/output/android-browser-staging-e2e/$run_id"
 tools_image="who-need-help:android-e2e-tools-$run_id"
@@ -24,6 +30,51 @@ prepared=0
 android_runner_pid=
 android_runner_output=
 
+case "$#" in
+  0) ;;
+  2)
+    if [[ "$1" != --remote ]]; then
+      echo "usage: $0 [--remote SSH_HOST:/absolute/test/checkout]" >&2
+      exit 1
+    fi
+    remote_target=$2
+    ;;
+  *)
+    echo "usage: $0 [--remote SSH_HOST:/absolute/test/checkout]" >&2
+    exit 1
+    ;;
+esac
+
+if [[ -n "$remote_target" ]]; then
+  case "$remote_target" in
+    *:/*)
+      remote_host=${remote_target%%:*}
+      remote_root=${remote_target#*:}
+      ;;
+    *)
+      echo "The remote target must be SSH_HOST:/absolute/test/checkout." >&2
+      exit 1
+      ;;
+  esac
+
+  case "$remote_host" in
+    "" | -* | *[!A-Za-z0-9._@-]*)
+      echo "The remote SSH host contains unsupported characters." >&2
+      exit 1
+      ;;
+  esac
+
+  case "$remote_root" in
+    /) echo "The remote checkout must not be the filesystem root." >&2; exit 1 ;;
+    /*/../* | */.. | *[!A-Za-z0-9._/-]*)
+      echo "The remote checkout path is not accepted." >&2
+      exit 1
+      ;;
+    /*) ;;
+    *) echo "The remote checkout path must be absolute." >&2; exit 1 ;;
+  esac
+fi
+
 if [[ ! -e /dev/kvm ]]; then
   echo "/dev/kvm is required for the containerized Android emulator." >&2
   exit 1
@@ -49,17 +100,77 @@ set -a
 . "$ENV_FILE"
 set +a
 
-if [[ "${DATABASE_MODE:-container}" != container ]]; then
+if [[ -z "$remote_target" && "${DATABASE_MODE:-container}" != container ]]; then
   echo "This rollback-based Android/browser drill requires DATABASE_MODE=container." >&2
   exit 1
 fi
 
-: "${POSTGRES_DB:?POSTGRES_DB is missing from .env}"
+if [[ -n "$remote_target" ]]; then
-: "${WNH_BASE_URL:?WNH_BASE_URL is missing from .env}"
+  mapfile -t remote_metadata < <(
-: "${WNH_TRACKING_MIN_TIME_MS:?WNH_TRACKING_MIN_TIME_MS is missing from .env}"
+    ssh "$remote_host" bash -s -- "$remote_root" <<'REMOTE_METADATA'
-: "${WNH_TRACKING_HTTP_TIMEOUT_MS:?WNH_TRACKING_HTTP_TIMEOUT_MS is missing from .env}"
+set -euo pipefail
-: "${MAILPIT_BIND_ADDRESS:?MAILPIT_BIND_ADDRESS is missing from .env}"
+root=$1
-: "${MAILPIT_PORT:?MAILPIT_PORT is missing from .env}"
+cd "$root"
+if [[ ! -f .env ]]; then
+  echo "Missing $root/.env." >&2
+  exit 1
+fi
+set -a
+# shellcheck source=/dev/null
+. ./.env
+set +a
+: "${DEPLOYMENT_ENV:?DEPLOYMENT_ENV is missing from remote .env}"
+: "${DEPLOYMENT_TARGET:?DEPLOYMENT_TARGET is missing from remote .env}"
+: "${DATABASE_MODE:?DATABASE_MODE is missing from remote .env}"
+: "${POSTGRES_DB:?POSTGRES_DB is missing from remote .env}"
+: "${WNH_BASE_URL:?WNH_BASE_URL is missing from remote .env}"
+: "${WNH_TRACKING_MIN_TIME_MS:?WNH_TRACKING_MIN_TIME_MS is missing from remote .env}"
+: "${WNH_TRACKING_HTTP_TIMEOUT_MS:?WNH_TRACKING_HTTP_TIMEOUT_MS is missing from remote .env}"
+: "${MAILPIT_BIND_ADDRESS:?MAILPIT_BIND_ADDRESS is missing from remote .env}"
+: "${MAILPIT_PORT:?MAILPIT_PORT is missing from remote .env}"
+: "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is missing from remote .env}"
+if [[ "$DEPLOYMENT_ENV" != test || "$DEPLOYMENT_TARGET" != compose || "$DATABASE_MODE" != container ]]; then
+  echo "Remote cross-client E2E accepts only a container-backed test deployment." >&2
+  exit 1
+fi
+printf '%s\n' \
+  "$POSTGRES_DB" \
+  "$WNH_BASE_URL" \
+  "$WNH_TRACKING_MIN_TIME_MS" \
+  "$WNH_TRACKING_HTTP_TIMEOUT_MS" \
+  "$MAILPIT_BIND_ADDRESS" \
+  "$MAILPIT_PORT" \
+  "$COMPOSE_PROJECT_NAME"
+REMOTE_METADATA
+  )
+
+  if [[ "${#remote_metadata[@]}" -ne 7 ]]; then
+    echo "Could not read the expected non-secret metadata from the remote test deployment." >&2
+    exit 1
+  fi
+
+  POSTGRES_DB=${remote_metadata[0]}
+  WNH_BASE_URL=${remote_metadata[1]}
+  WNH_TRACKING_MIN_TIME_MS=${remote_metadata[2]}
+  WNH_TRACKING_HTTP_TIMEOUT_MS=${remote_metadata[3]}
+  MAILPIT_BIND_ADDRESS=${remote_metadata[4]}
+  MAILPIT_PORT=${remote_metadata[5]}
+  remote_compose_project=${remote_metadata[6]}
+  remote_output_dir="$remote_root/output/android-browser-staging-e2e/$run_id"
+  remote_tools_image="who-need-help:android-e2e-tools-$run_id"
+
+  if [[ "$remote_compose_project" != *_test ]]; then
+    echo "The remote Compose project is not marked as a test project." >&2
+    exit 1
+  fi
+fi
+
+: "${POSTGRES_DB:?POSTGRES_DB is missing from deployment config}"
+: "${WNH_BASE_URL:?WNH_BASE_URL is missing from deployment config}"
+: "${WNH_TRACKING_MIN_TIME_MS:?WNH_TRACKING_MIN_TIME_MS is missing from deployment config}"
+: "${WNH_TRACKING_HTTP_TIMEOUT_MS:?WNH_TRACKING_HTTP_TIMEOUT_MS is missing from deployment config}"
+: "${MAILPIT_BIND_ADDRESS:?MAILPIT_BIND_ADDRESS is missing from deployment config}"
+: "${MAILPIT_PORT:?MAILPIT_PORT is missing from deployment config}"
 
 if [[ ! "$MAILPIT_PORT" =~ ^[0-9]+$ ]] ||
   ((MAILPIT_PORT < 1 || MAILPIT_PORT > 65535)); then
@@ -93,6 +204,26 @@ mailpit_url="http://${mailpit_host}:${MAILPIT_PORT}"
 mkdir -p "$output_dir"
 chmod 700 "$ROOT/output" "$ROOT/output/android-browser-staging-e2e" "$output_dir"
 
+if [[ -n "$remote_target" ]]; then
+  remote_runtime=$(
+    ssh "$remote_host" bash -s -- "$remote_root" <<'REMOTE_RUNTIME'
+set -euo pipefail
+cd "$1"
+db_container=$(docker compose --env-file .env ps -q db)
+[[ -n "$db_container" ]]
+internal_network_id=$(
+  docker inspect "$db_container" |
+    jq -r '.[0].NetworkSettings.Networks | to_entries[] | select(.key | endswith("_internal")) | .value.NetworkID' |
+    head -n 1
+)
+[[ -n "$internal_network_id" ]]
+printf '%s\n%s\n' "$db_container" "$internal_network_id"
+REMOTE_RUNTIME
+  )
+  mapfile -t remote_runtime_lines <<<"$remote_runtime"
+  db_container=${remote_runtime_lines[0]:-}
+  internal_network_id=${remote_runtime_lines[1]:-}
+else
   db_container=$(docker compose ps -q db)
   if [[ -z "$db_container" ]]; then
     echo "The ordinary Compose database container is not running." >&2
@@ -104,6 +235,7 @@ internal_network_id=$(
       jq -r '.[0].NetworkSettings.Networks | to_entries[] | select(.key | endswith("_internal")) | .value.NetworkID' |
       head -n 1
   )
+fi
 
 if [[ -z "$internal_network_id" ]]; then
   echo "Could not identify the ordinary Compose internal network." >&2
@@ -113,9 +245,23 @@ fi
 snapshot_database() {
   local destination=$1
 
+  if [[ -n "$remote_target" ]]; then
+    # The expanded path is restricted to absolute [A-Za-z0-9._/-] above.
+    # shellcheck disable=SC2029
+    snapshot_query |
+      ssh "$remote_host" \
+        "cd $remote_root && docker compose --env-file .env exec -T db sh -c 'psql --no-psqlrc --set ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\"'" \
+        >"$destination"
+  else
+    snapshot_query |
       docker compose exec -T db sh -c \
         'psql --no-psqlrc --set ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' \
-    >"$destination" <<'SQL'
+        >"$destination"
+  fi
+}
+
+snapshot_query() {
+  cat <<'SQL'
 BEGIN READ ONLY;
 SELECT 'users' AS table_name, count(*) AS row_count FROM users
 UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
@@ -144,6 +290,79 @@ SQL
 run_fixture_tool() {
   local action=$1
 
+  if [[ -n "$remote_target" ]]; then
+    ssh "$remote_host" bash -s -- \
+      "$remote_root" \
+      "$action" \
+      "$internal_network_id" \
+      "$remote_tools_image" \
+      "$remote_output_dir" \
+      "$POSTGRES_DB" \
+      "$run_id" \
+      "$fixture_password" \
+      "$android_message" \
+      "$browser_reply" <<'REMOTE_FIXTURE'
+set -euo pipefail
+root=$1
+action=$2
+internal_network_id=$3
+tools_image=$4
+output_dir=$5
+postgres_db=$6
+run_id=$7
+fixture_password=$8
+android_message=$9
+browser_reply=${10}
+cd "$root"
+docker run --rm \
+  --network "$internal_network_id" \
+  --env-file .env \
+  --env APP_ROLE=migrate \
+  --env "WNH_ANDROID_E2E_EXPECTED_DATABASE=$postgres_db" \
+  --env WNH_ANDROID_E2E_CONFIRM=public-staging-android-e2e \
+  --env "WNH_ANDROID_E2E_RUN_ID=$run_id" \
+  --env "WNH_ANDROID_E2E_PASSWORD=$fixture_password" \
+  --env "WNH_ANDROID_E2E_ANDROID_MESSAGE=$android_message" \
+  --env "WNH_ANDROID_E2E_BROWSER_REPLY=$browser_reply" \
+  --env WNH_ANDROID_E2E_MANIFEST_PATH=/output/fixture.json \
+  --volume "$output_dir:/output" \
+  "$tools_image" \
+  mix wnh.staging_android_e2e "$action"
+REMOTE_FIXTURE
+
+    if [[ "$action" == prepare ]]; then
+      prepared=1
+    fi
+
+    if [[ "$action" == prepare || "$action" == verify ]]; then
+      ssh "$remote_host" bash -s -- \
+        "$remote_output_dir" "$remote_tools_image" <<'REMOTE_FIXTURE_OWNER'
+set -euo pipefail
+output_dir=$1
+tools_image=$2
+owner_uid=$(id -u)
+owner_gid=$(id -g)
+docker run --rm \
+  --volume "$output_dir:/output" \
+  --entrypoint sh \
+  "$tools_image" \
+  -euc "chown $owner_uid:$owner_gid /output/fixture*.json; chmod 600 /output/fixture*.json"
+REMOTE_FIXTURE_OWNER
+    fi
+
+    case "$action" in
+      prepare)
+        scp -q "$remote_host:$remote_output_dir/fixture.json" \
+          "$output_dir/fixture.json"
+        chmod 600 "$output_dir/fixture.json"
+        ;;
+      verify)
+        scp -q "$remote_host:$remote_output_dir/fixture-verification.json" \
+          "$output_dir/fixture-verification.json"
+        chmod 600 "$output_dir/fixture-verification.json"
+        ;;
+    esac
+  else
     docker run --rm \
       --network "$internal_network_id" \
       --env-file "$ENV_FILE" \
@@ -158,6 +377,10 @@ run_fixture_tool() {
       --volume "$output_dir:/output" \
       "$tools_image" \
       mix wnh.staging_android_e2e "$action"
+    if [[ "$action" == prepare ]]; then
+      prepared=1
+    fi
+  fi
 }
 
 adb() {
@@ -255,7 +478,33 @@ cleanup() {
 
   docker rm -f "$container" >/dev/null 2>&1 || true
   docker volume rm -f "$avd_volume" >/dev/null 2>&1 || true
-  docker image rm "$android_image" "$browser_image" "$tools_image" >/dev/null 2>&1 || true
+  if [[ -n "$remote_target" ]]; then
+    ssh "$remote_host" bash -s -- \
+      "$remote_root" "$remote_output_dir" "$remote_tools_image" \
+      >"$output_dir/remote-cleanup.log" 2>&1 <<'REMOTE_CLEANUP' || status=1
+set -euo pipefail
+root=$1
+output_dir=$2
+tools_image=$3
+case "$output_dir" in
+  "$root"/output/android-browser-staging-e2e/*) ;;
+  *) echo "Refusing unexpected remote output path." >&2; exit 1 ;;
+esac
+docker image rm "$tools_image" >/dev/null 2>&1 || true
+if [[ -d "$output_dir" && ! -L "$output_dir" ]]; then
+  find "$output_dir" -mindepth 1 -maxdepth 1 -type f -delete
+  rmdir "$output_dir"
+fi
+REMOTE_CLEANUP
+  else
+    docker image rm "$tools_image" >/dev/null 2>&1 || true
+  fi
+  docker image rm "$android_image" "$browser_image" >/dev/null 2>&1 || true
+
+  if [[ -n "$ssh_tunnel_pid" ]]; then
+    kill "$ssh_tunnel_pid" >/dev/null 2>&1 || true
+    wait "$ssh_tunnel_pid" >/dev/null 2>&1 || true
+  fi
 
   {
     printf 'container_absent='
@@ -277,6 +526,15 @@ cleanup() {
     else
       printf 'unknown\n'
     fi
+    if [[ -n "$remote_target" ]]; then
+      printf 'remote_test_target=%s\n' "$remote_target"
+      printf 'mailpit_tunnel_stopped='
+      if [[ -n "$ssh_tunnel_pid" ]] && kill -0 "$ssh_tunnel_pid" 2>/dev/null; then
+        printf 'false\n'
+      else
+        printf 'true\n'
+      fi
+    fi
   } >"$output_dir/cleanup.txt"
 
   unset fixture_password
@@ -284,14 +542,70 @@ cleanup() {
 }
 trap cleanup EXIT HUP INT TERM
 
+if [[ -n "$remote_target" ]]; then
+  local_mailpit_port=
+  for candidate_port in $(seq 18028 18068); do
+    if ! ss -ltnH "sport = :$candidate_port" | grep -q .; then
+      local_mailpit_port=$candidate_port
+      break
+    fi
+  done
+  if [[ -z "$local_mailpit_port" ]]; then
+    echo "Could not allocate a loopback port for the remote Mailpit tunnel." >&2
+    exit 1
+  fi
+
+  ssh -o ExitOnForwardFailure=yes -N \
+    -L "127.0.0.1:$local_mailpit_port:127.0.0.1:$MAILPIT_PORT" \
+    "$remote_host" >"$output_dir/mailpit-tunnel.log" 2>&1 &
+  ssh_tunnel_pid=$!
+  mailpit_url="http://127.0.0.1:$local_mailpit_port"
+
+  mailpit_ready=0
+  for _attempt in $(seq 1 30); do
+    if ! kill -0 "$ssh_tunnel_pid" 2>/dev/null; then
+      break
+    fi
+    if curl --fail --silent --show-error "$mailpit_url/api/v1/info" \
+      >"$output_dir/mailpit-info.json" 2>/dev/null; then
+      mailpit_ready=1
+      break
+    fi
+    sleep 1
+  done
+  if [[ "$mailpit_ready" -ne 1 ]]; then
+    echo "The SSH tunnel to the remote test Mailpit did not become ready." >&2
+    exit 1
+  fi
+fi
+
 curl --fail --silent --show-error "$WNH_BASE_URL/healthz/ready" \
   >"$output_dir/public-ready.txt"
 snapshot_database "$output_dir/database-before.txt"
 sha256sum "$APK" >"$output_dir/apk.sha256"
 sha256sum "$TEST_APK" >"$output_dir/test-apk.sha256"
 
+if [[ -n "$remote_target" ]]; then
+  ssh "$remote_host" bash -s -- \
+    "$remote_root" "$remote_output_dir" "$remote_tools_image" \
+    >"$output_dir/tools-build.log" 2>&1 <<'REMOTE_TOOLS_BUILD'
+set -euo pipefail
+root=$1
+output_dir=$2
+tools_image=$3
+case "$output_dir" in
+  "$root"/output/android-browser-staging-e2e/*) ;;
+  *) echo "Refusing unexpected remote output path." >&2; exit 1 ;;
+esac
+mkdir -p "$output_dir"
+chmod 700 "$output_dir"
+cd "$root"
+docker build --target load_tools --tag "$tools_image" .
+REMOTE_TOOLS_BUILD
+else
   docker build --target load_tools --tag "$tools_image" . \
     >"$output_dir/tools-build.log"
+fi
 docker build --tag "$browser_image" e2e \
   >"$output_dir/browser-build.log"
 docker build \
@@ -382,12 +696,13 @@ if [[ "$network_ready" -ne 1 ]]; then
 fi
 
 run_fixture_tool prepare >"$output_dir/fixture-prepare.log"
-prepared=1
+if [[ -z "$remote_target" ]]; then
   docker run --rm \
     --volume "$output_dir:/output" \
     --entrypoint sh \
     "$tools_image" \
     -euc "chown $(id -u):$(id -g) /output/fixture.json; chmod 600 /output/fixture.json"
+fi
 
 request_path=$(jq -r '.request.path' "$output_dir/fixture.json")
 helper_login_path=$(jq -r '.helper_login_path' "$output_dir/fixture.json")
@@ -441,11 +756,13 @@ fi
 
 run_browser_phase observe_stopped
 run_fixture_tool verify >"$output_dir/fixture-verify.log"
+if [[ -z "$remote_target" ]]; then
   docker run --rm \
     --volume "$output_dir:/output" \
     --entrypoint sh \
     "$tools_image" \
     -euc "chown $(id -u):$(id -g) /output/fixture-verification.json; chmod 600 /output/fixture-verification.json"
+fi
 
 adb logcat -d -s WhoNeedHelpWebView:D AndroidRuntime:E '*:S' \
   >"$output_dir/logcat-before-cleanup.txt"