test: verify active realtime failover

.env.e2e.example

@@ -11,9 +11,9 @@ TRAEFIK_DOCKER_NETWORK=GENERATED_UNIQUE_E2E_NETWORK
 TRAEFIK_ROUTER_RULE='PathPrefix(`/`)'
 
 PHX_HOST=proxy
-PHX_SCHEME=http
+PHX_SCHEME=https
-PHX_URL_PORT=80
+PHX_URL_PORT=443
-MAP_TILE_URL=http://proxy/__e2e__/map-tile.png?z={z}&x={x}&y={y}
+MAP_TILE_URL=https://proxy/__e2e__/map-tile.png?z={z}&x={x}&y={y}
 
 POSTGRES_DB=who_need_help_e2e
 POSTGRES_USER=postgres
@@ -47,6 +47,6 @@ GITHUB_OAUTH_USER_URL=
 GITHUB_OAUTH_HTTP_CONNECT_TIMEOUT_MS=
 GITHUB_OAUTH_HTTP_RECEIVE_TIMEOUT_MS=
 
-E2E_BASE_URL=http://proxy
+E2E_BASE_URL=https://proxy
 E2E_MAILPIT_URL=http://mailpit:8025
 E2E_OUTPUT_DIR=GENERATED_ABSOLUTE_IGNORED_OUTPUT_DIRECTORY

Dockerfile

@@ -55,8 +55,8 @@ COPY mix.exs mix.lock ./
 RUN mix deps.get --only $MIX_ENV
 RUN mkdir config
 
-ARG WNH_E2E_SSL_EXCLUDE_HOST=
+ARG WNH_E2E_ROUTES=false
-ENV WNH_E2E_SSL_EXCLUDE_HOST="${WNH_E2E_SSL_EXCLUDE_HOST}"
+ENV WNH_E2E_ROUTES="${WNH_E2E_ROUTES}"
 
 # copy compile-time config files before we compile dependencies
 # to ensure any relevant config change will trigger the dependencies

README.md

@@ -232,9 +232,13 @@ two worker replicas:
 ```
 
 On its first run it generates `.env.e2e` with independent random local secrets
-and mode `0600`. The suite registers users through real Mailpit messages and
+and mode `0600`. Browser traffic uses the isolated Traefik HTTPS entrypoint;
-uses the audited one-time bootstrap command inside only the isolated E2E
+Playwright accepts only that one-run proxy's generated certificate. E2E-only
-database to create its moderator. It covers public/authentication boundaries;
+fixture and failure-injection routes are enabled by a compile-time flag that is
+disabled in the ordinary production image. The suite registers users through
+real Mailpit messages and uses the audited one-time bootstrap command inside
+only the isolated E2E database to create its moderator. It covers
+public/authentication boundaries;
 the urgent medicine flow through matching, realtime chat, handover, and
 double-blind reviews; and the Activity flow through join approval, private
 group chat, message-scoped reporting, blocking, privacy defaults, an unverified
@@ -244,6 +248,11 @@ four public pages in both themes, horizontal overflow at three viewport widths,
 an actual locally served raster map tile, LiveView offline/reconnect UI, public
 RU/UK locale persistence, and the selected Ukrainian locale in an authenticated
 LiveView.
+The active failover scenario grants real browser geolocation permission,
+starts consent-based tracking and chat, halts the exact BEAM node serving the
+helper, proves a disconnect and a different runtime boot identity, then checks
+tracking/chat recovery and raw-position deletion in Chromium, Firefox, and
+WebKit.
 An optional list of Playwright spec paths can be passed after the command for a
 focused diagnostic run.
 It retains traces, screenshots, video, and Compose logs under the ignored

compose.e2e.yaml

@@ -4,11 +4,17 @@ services:
     pull_policy: never
     build:
       args:
-        WNH_E2E_SSL_EXCLUDE_HOST: proxy
+        WNH_E2E_ROUTES: "true"
 
   web:
     image: who-need-help:e2e
     pull_policy: never
+    labels:
+      - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.rule=${TRAEFIK_ROUTER_RULE}
+      - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.entrypoints=websecure
+      - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.service=${TRAEFIK_APP_NAME}
+      - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.middlewares=${TRAEFIK_APP_NAME}-retry
+      - traefik.http.routers.${TRAEFIK_APP_NAME}-tls.tls=true
 
   worker:
     image: who-need-help:e2e

config/prod.exs

@@ -11,22 +11,21 @@ config :who_need_help, WhoNeedHelpWeb.Endpoint,
 # Force using SSL in production. This also sets the "strict-security-transport" header,
 # known as HSTS. If you have a health check endpoint, you may want to exclude it below.
 # Note `:force_ssl` is required to be set at compile-time.
-force_ssl_hosts =
+e2e_routes =
-  case System.get_env("WNH_E2E_SSL_EXCLUDE_HOST") do
+  case System.get_env("WNH_E2E_ROUTES", "false") do
-    nil -> ["localhost", "127.0.0.1"]
+    "true" -> true
-    "" -> ["localhost", "127.0.0.1"]
+    "false" -> false
-    "proxy" -> ["localhost", "127.0.0.1", "proxy"]
+    other -> raise "WNH_E2E_ROUTES must be true or false, got: #{inspect(other)}"
-    other -> raise "unsupported WNH_E2E_SSL_EXCLUDE_HOST: #{inspect(other)}"
   end
 
-config :who_need_help, :e2e_routes, System.get_env("WNH_E2E_SSL_EXCLUDE_HOST") == "proxy"
+config :who_need_help, :e2e_routes, e2e_routes
 
 config :who_need_help, WhoNeedHelpWeb.Endpoint,
   force_ssl: [
     rewrite_on: [:x_forwarded_proto],
     exclude: [
       paths: ["/healthz/live", "/healthz/ready"],
-      hosts: force_ssl_hosts
+      hosts: ["localhost", "127.0.0.1"]
     ]
   ]
 

e2e/playwright.bootstrap.config.ts

@@ -19,6 +19,7 @@ export default defineConfig({
   reporter: [["line"]],
   use: {
     baseURL,
+    ignoreHTTPSErrors: true,
     ...devices["Desktop Chrome"],
     actionTimeout: 10_000,
     navigationTimeout: 20_000,

e2e/playwright.config.ts

@@ -24,6 +24,7 @@ export default defineConfig({
   ],
   use: {
     baseURL,
+    ignoreHTTPSErrors: true,
     actionTimeout: 10_000,
     navigationTimeout: 20_000,
     trace: "retain-on-failure",

e2e/tests/active-failover.spec.ts

@@ -0,0 +1,215 @@
+import { expect, test } from "@playwright/test";
+import {
+  projectEmail,
+  projectText,
+  registerAndConfirm,
+  selectOptionContaining,
+  waitForLiveViewConnected,
+} from "./helpers";
+
+async function markerKinds(page: import("@playwright/test").Page): Promise<string[]> {
+  const encoded = await page.locator("#show-map").getAttribute("data-markers");
+  const markers = JSON.parse(encoded ?? "[]") as Array<{ title?: string }>;
+  return markers.map(({ title }) => title ?? "");
+}
+
+async function positionCount(page: import("@playwright/test").Page): Promise<number> {
+  const raw = await page.locator("#e2e-runtime-node").getAttribute("data-position-count");
+  return Number(raw ?? "-1");
+}
+
+async function runtimeSignature(
+  page: import("@playwright/test").Page,
+): Promise<{ node: string; bootID: string } | null> {
+  const runtime = page.locator("#e2e-runtime-node");
+  const [node, bootID] = await Promise.all([
+    runtime.getAttribute("data-node"),
+    runtime.getAttribute("data-boot-id"),
+  ]);
+
+  return node && bootID ? { node, bootID } : null;
+}
+
+test("active chat and browser tracking recover after the serving BEAM node restarts", async ({
+  browser,
+  request,
+}, testInfo) => {
+  const requester = await registerAndConfirm(
+    browser,
+    request,
+    projectEmail("failover-requester", testInfo.project.name),
+    "Failover Requester",
+  );
+  const helper = await registerAndConfirm(
+    browser,
+    request,
+    projectEmail("failover-helper", testInfo.project.name),
+    "Failover Helper",
+  );
+  const requestTitle = projectText("Failover medicine pickup", testInfo.project.name);
+
+  await requester.page.goto("/requests/new");
+  await selectOptionContaining(requester.page, "Category", "Medicine pickup");
+  await requester.page.getByLabel("Medicine pickup status").selectOption("reserved");
+  await requester.page.getByLabel("Short title").fill(requestTitle);
+  await requester.page
+    .getByLabel("What help do you need?")
+    .fill("A run-scoped reserved medicine pickup for failover verification.");
+  await requester.page.getByLabel("Urgency").selectOption("now");
+  await requester.page
+    .getByLabel("Request expires (UTC)")
+    .fill(new Date(Date.now() + 3 * 60 * 60 * 1000).toISOString().slice(0, 16));
+  await requester.page.getByLabel("Approximate area").fill("Failover district");
+  await requester.page.getByLabel("Location visibility").selectOption("approximate_public");
+  await requester.page.getByLabel("Latitude").fill("50.4501");
+  await requester.page.getByLabel("Longitude").fill("30.5234");
+  await requester.page.locator("#request-form input[type=checkbox]").check();
+  await requester.page.getByRole("button", { name: "Publish request" }).click();
+  await expect(requester.page).toHaveURL(/\/requests\/[0-9a-f-]+$/);
+
+  const requestURL = requester.page.url();
+  const origin = new URL(process.env.BASE_URL!).origin;
+  await helper.context.grantPermissions(["geolocation"], { origin });
+  await helper.context.setGeolocation({
+    latitude: 50.4504,
+    longitude: 30.5237,
+    accuracy: 5,
+  });
+  await helper.page.goto(requestURL);
+  await helper.page.getByRole("button", { name: "I can help" }).click();
+  await expect(helper.page.getByRole("heading", { name: "Private match chat" })).toBeVisible();
+  await expect(requester.page.getByRole("heading", { name: "Private match chat" })).toBeVisible();
+
+  const browserPosition = await helper.page.evaluate(
+    () =>
+      new Promise<{ latitude: number; longitude: number }>((resolve, reject) => {
+        navigator.geolocation.getCurrentPosition(
+          ({ coords }) =>
+            resolve({
+              latitude: coords.latitude,
+              longitude: coords.longitude,
+            }),
+          ({ code, message }) => reject(new Error(`geolocation ${code}: ${message}`)),
+        );
+      }),
+  );
+  expect(browserPosition.latitude).toBeCloseTo(50.4504, 4);
+  expect(browserPosition.longitude).toBeCloseTo(30.5237, 4);
+
+  await helper.page.getByRole("button", { name: "Share location" }).click();
+  await expect(helper.page.getByRole("button", { name: "Stop and delete position" })).toBeVisible();
+  await expect.poll(() => positionCount(helper.page)).toBe(1);
+  await expect.poll(() => positionCount(requester.page)).toBe(1);
+  await expect.poll(() => markerKinds(requester.page)).toContain("Shared live location");
+
+  await helper.page
+    .getByPlaceholder("Write a safe coordination message…")
+    .fill("Message before the serving node restart.");
+  await helper.page.getByRole("button", { name: "Send", exact: true }).click();
+  await expect(requester.page.getByText("Message before the serving node restart.")).toBeVisible();
+
+  await helper.page.evaluate(() => {
+    const state = { sawDisconnected: false };
+
+    (
+      window as typeof window & {
+        __wnhFailoverState?: { sawDisconnected: boolean };
+      }
+    ).__wnhFailoverState = state;
+
+    window.addEventListener("phx:page-loading-start", (event) => {
+      const detail = (event as CustomEvent<{ kind?: string }>).detail;
+
+      if (detail?.kind === "error") {
+        state.sawDisconnected = true;
+      }
+    });
+  });
+
+  const originalRuntime = await runtimeSignature(helper.page);
+  expect(originalRuntime).not.toBeNull();
+
+  const crash = await request.post("/__e2e__/crash-node", {
+    data: {
+      confirmation: "crash-exact-e2e-node",
+      node: originalRuntime!.node,
+    },
+  });
+  expect(crash.status()).toBe(202);
+  expect(await crash.json()).toMatchObject({
+    status: "scheduled",
+    node: originalRuntime!.node,
+  });
+
+  await expect
+    .poll(
+      () =>
+        helper.page.evaluate(
+          () =>
+            (
+              window as typeof window & {
+                __wnhFailoverState?: { sawDisconnected: boolean };
+              }
+            ).__wnhFailoverState?.sawDisconnected ?? false,
+        ),
+      { timeout: 30_000 },
+    )
+    .toBe(true);
+
+  await waitForLiveViewConnected(helper.page);
+  let recoveredRuntime: Awaited<ReturnType<typeof runtimeSignature>> = null;
+  await expect
+    .poll(
+      async () => {
+        recoveredRuntime = await runtimeSignature(helper.page);
+        return recoveredRuntime
+          ? `${recoveredRuntime.node}/${recoveredRuntime.bootID}`
+          : null;
+      },
+      { timeout: 30_000 },
+    )
+    .not.toBe(`${originalRuntime!.node}/${originalRuntime!.bootID}`);
+
+  await expect(helper.page.getByRole("button", { name: "Stop and delete position" })).toBeVisible();
+  await expect.poll(() => positionCount(helper.page), { timeout: 30_000 }).toBe(1);
+  await expect.poll(() => positionCount(requester.page), { timeout: 30_000 }).toBe(1);
+  await expect.poll(() => markerKinds(requester.page), { timeout: 30_000 }).toContain(
+    "Shared live location",
+  );
+
+  await helper.page
+    .getByPlaceholder("Write a safe coordination message…")
+    .fill("Message after LiveView recovered on the available replicas.");
+  await helper.page.getByRole("button", { name: "Send", exact: true }).click();
+  await expect(
+    requester.page.getByText("Message after LiveView recovered on the available replicas."),
+  ).toBeVisible();
+
+  await helper.page.getByRole("button", { name: "Stop and delete position" }).click();
+  await expect(helper.page.getByRole("button", { name: "Share location" })).toBeVisible();
+  await expect.poll(() => positionCount(helper.page)).toBe(0);
+  await expect.poll(() => positionCount(requester.page)).toBe(0);
+  await expect.poll(() => markerKinds(requester.page)).not.toContain("Shared live location");
+
+  await testInfo.attach("active-failover-evidence", {
+    contentType: "application/json",
+    body: Buffer.from(
+      JSON.stringify(
+        {
+          browser: testInfo.project.name,
+          original_runtime: originalRuntime,
+          recovered_runtime: recoveredRuntime,
+          saw_liveview_disconnect: true,
+          tracking_recovered: true,
+          chat_recovered: true,
+          tracking_position_deleted: true,
+        },
+        null,
+        2,
+      ),
+    ),
+  });
+
+  await requester.context.close();
+  await helper.context.close();
+});

lib/who_need_help/application.ex

@@ -5,8 +5,15 @@ defmodule WhoNeedHelp.Application do
 
   use Application
 
+  @e2e_routes Application.compile_env(:who_need_help, :e2e_routes, false)
+
   @impl true
   def start(_type, _args) do
+    if @e2e_routes do
+      boot_id = :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false)
+      :persistent_term.put({WhoNeedHelp, :e2e_boot_id}, boot_id)
+    end
+
     common_children = [
       WhoNeedHelpWeb.Telemetry,
       WhoNeedHelp.Repo,

lib/who_need_help_web/controllers/e2e_failure_controller.ex

@@ -0,0 +1,31 @@
+defmodule WhoNeedHelpWeb.E2eFailureController do
+  @moduledoc false
+
+  use WhoNeedHelpWeb, :controller
+
+  @confirmation "crash-exact-e2e-node"
+
+  def crash_node(conn, %{"confirmation" => @confirmation, "node" => target}) do
+    current_nodes = [Node.self() | Node.list()]
+
+    case Enum.find(current_nodes, &(to_string(&1) == target)) do
+      nil ->
+        conn
+        |> put_status(:not_found)
+        |> json(%{error: "unknown_node"})
+
+      target_node ->
+        true = :rpc.cast(target_node, :timer, :apply_after, [500, :erlang, :halt, [1]])
+
+        conn
+        |> put_status(:accepted)
+        |> json(%{status: "scheduled", node: target})
+    end
+  end
+
+  def crash_node(conn, _params) do
+    conn
+    |> put_status(:unprocessable_entity)
+    |> json(%{error: "invalid_confirmation"})
+  end
+end

lib/who_need_help_web/live/request_live/show.ex

@@ -3,6 +3,8 @@ defmodule WhoNeedHelpWeb.RequestLive.Show do
 
   alias WhoNeedHelp.{Help, Messaging, Tracking, Trust}
 
+  @e2e_routes Application.compile_env(:who_need_help, :e2e_routes, false)
+
   @impl true
   def mount(%{"id" => id}, _session, socket) do
     case Help.get_request(socket.assigns.current_scope, id) do
@@ -15,6 +17,17 @@ defmodule WhoNeedHelpWeb.RequestLive.Show do
           socket
           |> assign(:subscribed_assignment_id, nil)
           |> assign(:native_client, native_client)
+          |> assign(
+            :e2e_runtime_node,
+            if(@e2e_routes && connected?(socket), do: to_string(Node.self()), else: nil)
+          )
+          |> assign(
+            :e2e_boot_id,
+            if(@e2e_routes && connected?(socket),
+              do: :persistent_term.get({WhoNeedHelp, :e2e_boot_id}),
+              else: nil
+            )
+          )
 
         if connected?(socket), do: Help.subscribe_request(id)
 
@@ -548,6 +561,16 @@ defmodule WhoNeedHelpWeb.RequestLive.Show do
   def render(assigns) do
     ~H"""
     <Layouts.app flash={@flash} current_scope={@current_scope}>
+      <div
+        :if={@e2e_runtime_node}
+        id="e2e-runtime-node"
+        data-node={@e2e_runtime_node}
+        data-boot-id={@e2e_boot_id}
+        data-position-count={map_size(@positions)}
+        hidden
+        aria-hidden="true"
+      >
+      </div>
       <.link navigate={~p"/requests"} class="btn btn-ghost btn-sm">
         {gettext("← All requests")}
       </.link>

lib/who_need_help_web/router.ex

@@ -46,6 +46,7 @@ defmodule WhoNeedHelpWeb.Router do
       pipe_through :api
 
       get "/map-tile.png", E2eTileController, :show
+      post "/crash-node", E2eFailureController, :crash_node
     end
   end
 

scripts/e2e-run.sh

@@ -32,7 +32,7 @@ export CODEX_SESSION_ID="${CODEX_SESSION_ID:-${CODEX_THREAD_ID:-local-e2e}}"
 # Keep raster-map traffic inside the isolated application network. MapLibre
 # performs tile fetches from a worker, so browser-context request interception
 # is not a reliable network boundary for these requests.
-export MAP_TILE_URL="http://proxy/__e2e__/map-tile.png?z={z}&x={x}&y={y}"
+export MAP_TILE_URL="https://proxy/__e2e__/map-tile.png?z={z}&x={x}&y={y}"
 
 compose() {
   docker compose \

scripts/ensure-local-e2e-env.sh

@@ -7,11 +7,27 @@ target="$ROOT/.env.e2e"
 
 if [ -f "$target" ]; then
   chmod 600 "$target"
+  updated=false
 
   if ! grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$target"; then
     printf '\nTRAEFIK_RETRY_ATTEMPTS=3\n' >>"$target"
+    updated=true
+  fi
+
+  if grep -q '^E2E_BASE_URL=http://proxy$' "$target"; then
+    sed -i \
+      -e 's|^E2E_BASE_URL=http://proxy$|E2E_BASE_URL=https://proxy|' \
+      -e 's|^PHX_SCHEME=http$|PHX_SCHEME=https|' \
+      -e 's|^PHX_URL_PORT=80$|PHX_URL_PORT=443|' \
+      -e 's|^MAP_TILE_URL=http://proxy/|MAP_TILE_URL=https://proxy/|' \
+      "$target"
+    updated=true
+  fi
+
   chmod 600 "$target"
-    echo "Added the missing Traefik retry input; no E2E secret was changed."
+
+  if [ "$updated" = true ]; then
+    echo "Updated non-secret E2E transport inputs; existing secrets were preserved."
   else
     echo ".env.e2e already exists; no secret or experiment input was changed."
   fi
@@ -35,9 +51,9 @@ TRAEFIK_APP_NAME=generated-per-run
 TRAEFIK_DOCKER_NETWORK=generated-per-run
 TRAEFIK_ROUTER_RULE='PathPrefix(\`/\`)'
 PHX_HOST=proxy
-PHX_SCHEME=http
+PHX_SCHEME=https
-PHX_URL_PORT=80
+PHX_URL_PORT=443
-MAP_TILE_URL=http://proxy/__e2e__/map-tile.png?z={z}&x={x}&y={y}
+MAP_TILE_URL=https://proxy/__e2e__/map-tile.png?z={z}&x={x}&y={y}
 POSTGRES_DB=who_need_help_e2e
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=$postgres_password
@@ -60,7 +76,7 @@ CODEX_SESSION_ID=local-e2e
 RATE_LIMIT_POLICIES_JSON={}
 GITHUB_OAUTH_CLIENT_ID=
 GITHUB_OAUTH_CLIENT_SECRET=
-E2E_BASE_URL=http://proxy
+E2E_BASE_URL=https://proxy
 E2E_MAILPIT_URL=http://mailpit:8025
 E2E_OUTPUT_DIR=$ROOT/output/e2e/generated-per-run
 EOF

test/who_need_help_web/router_boundary_test.exs

@@ -1,10 +1,10 @@
 defmodule WhoNeedHelpWeb.RouterBoundaryTest do
   use ExUnit.Case, async: true
 
-  test "the local raster fixture route is absent outside an E2E-compiled release" do
+  test "E2E-only routes are absent outside an E2E-compiled release" do
-    refute Enum.any?(
+    route_paths = MapSet.new(WhoNeedHelpWeb.Router.__routes__(), & &1.path)
-             WhoNeedHelpWeb.Router.__routes__(),
+
-             &(&1.path == "/__e2e__/map-tile.png")
+    refute MapSet.member?(route_paths, "/__e2e__/map-tile.png")
-           )
+    refute MapSet.member?(route_paths, "/__e2e__/crash-node")
   end
 end