Make production map E2E deterministic

e2e/tests/browser-failure-policy.spec.ts

@@ -1,10 +1,33 @@
 import { expect, test } from "@playwright/test";
 import {
+  installDeterministicMapTileRoute,
   isTransientNavigationError,
   isTransientOpenStreetMapTileFailure,
   reconcileTransientOpenStreetMapConsoleFailures,
 } from "./helpers";
 
+test("the production browser verifier uses a decodable deterministic raster tile", async ({
+  browser,
+}) => {
+  const context = await browser.newContext({ serviceWorkers: "block" });
+  await installDeterministicMapTileRoute(context);
+  const page = await context.newPage();
+
+  await page.setContent(
+    '<img alt="map tile" src="https://tile.openstreetmap.org/0/0/0.png">',
+  );
+
+  await expect(page.getByAltText("map tile")).toHaveJSProperty(
+    "complete",
+    true,
+  );
+  await expect(page.getByAltText("map tile")).toHaveJSProperty(
+    "naturalWidth",
+    1,
+  );
+  await context.close();
+});
+
 test("navigation retry covers bounded goto timeouts without treating application errors as transient", () => {
   expect(
     isTransientNavigationError(
@@ -91,7 +114,7 @@ test("tile console errors remain fatal without a matching transient request fail
     locationURL: "https://whoneedhelp.com/assets/js/app.js",
   };
 
-  expect(reconcileTransientOpenStreetMapConsoleFailures([failure], [])).toEqual([
-    failure,
-  ]);
+  expect(reconcileTransientOpenStreetMapConsoleFailures([failure], [])).toEqual(
+    [failure],
+  );
 });

e2e/tests/helpers.ts

@@ -21,6 +21,26 @@ export type AuthenticatedBrowser = {
   email: string;
 };
 
+const deterministicRasterTile = Buffer.from(
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
+  "base64",
+);
+const openStreetMapRasterTileURL =
+  /^https:\/\/tile\.openstreetmap\.org\/\d+\/\d+\/\d+\.png$/;
+
+export async function installDeterministicMapTileRoute(
+  context: BrowserContext,
+): Promise<void> {
+  await context.route(openStreetMapRasterTileURL, async (route) => {
+    await route.fulfill({
+      status: 200,
+      contentType: "image/png",
+      headers: { "cache-control": "public, max-age=604800" },
+      body: deterministicRasterTile,
+    });
+  });
+}
+
 export function projectEmail(localPart: string, projectName: string): string {
   const runID = process.env.E2E_RUN_ID;
 
@@ -36,8 +56,16 @@ export function projectText(value: string, projectName: string): string {
   return `${value} [${projectName}]`;
 }
 
-export async function newIsolatedContext(browser: Browser): Promise<BrowserContext> {
-  return browser.newContext({ serviceWorkers: "block" });
+export async function newIsolatedContext(
+  browser: Browser,
+): Promise<BrowserContext> {
+  const context = await browser.newContext({ serviceWorkers: "block" });
+
+  if (process.env.E2E_DETERMINISTIC_MAP_TILES === "1") {
+    await installDeterministicMapTileRoute(context);
+  }
+
+  return context;
 }
 
 export async function gotoWithTransientRetry(
@@ -165,7 +193,10 @@ export function reconcileTransientOpenStreetMapConsoleFailures(
 
     if (consumeGenericResourceFailure(text)) return false;
 
-    if (genericMapLibreFetchBudget > 0 && text === "TypeError: Failed to fetch") {
+    if (
+      genericMapLibreFetchBudget > 0 &&
+      text === "TypeError: Failed to fetch"
+    ) {
       genericMapLibreFetchBudget -= 1;
       return false;
     }
@@ -249,7 +280,9 @@ export async function setRequestLocation(
       .check();
   }
 
-  const manualCoordinates = workspace.locator("details.request-location-manual");
+  const manualCoordinates = workspace.locator(
+    "details.request-location-manual",
+  );
 
   if (!(await manualCoordinates.getAttribute("open"))) {
     await manualCoordinates.locator("summary").click();
@@ -458,7 +491,9 @@ export async function waitForApplicationEmailLink(
     .poll(
       async () => {
         messageID = await latestMessageID(request, email);
-        return messageID && messageID !== previousMessageID ? messageID : undefined;
+        return messageID && messageID !== previousMessageID
+          ? messageID
+          : undefined;
       },
       {
         message: `waiting for the Mailpit login message for ${email}`,
@@ -467,7 +502,9 @@ export async function waitForApplicationEmailLink(
     )
     .toBeTruthy();
 
-  const response = await request.get(`${mailpitURL}/api/v1/message/${messageID}`);
+  const response = await request.get(
+    `${mailpitURL}/api/v1/message/${messageID}`,
+  );
   expect(response.ok()).toBeTruthy();
 
   const message = (await response.json()) as {
@@ -487,7 +524,9 @@ export async function waitForApplicationEmailLink(
     });
 
   if (!match) {
-    throw new Error(`No application link starting with ${pathPrefix} found for ${email}`);
+    throw new Error(
+      `No application link starting with ${pathPrefix} found for ${email}`,
+    );
   }
 
   const link = new URL(match);

scripts/production-full-e2e.sh

@@ -351,6 +351,44 @@ curl --fail --silent --show-error --max-time 10 "$BASE_URL/healthz/ready" \
   >"$output_dir/readiness-before.json"
 snapshot "$output_dir/database-before.json"
 
+# OSM's public raster service is best-effort and must not be bulk-tested. Probe
+# exactly one canonical tile with an identifiable client and retain enough
+# evidence to distinguish an unavailable provider from an application failure.
+tile_headers="$output_dir/osm-tile-boundary.headers"
+tile_body="$output_dir/osm-tile-boundary.png"
+curl --fail --silent --show-error --max-time 10 \
+  --user-agent "WhoNeedHelp-production-e2e/1.0 (+https://whoneedhelp.com)" \
+  --referer "$BASE_URL/" \
+  --dump-header "$tile_headers" \
+  --output "$tile_body" \
+  https://tile.openstreetmap.org/0/0/0.png
+tile_status=$(awk 'toupper($1) ~ /^HTTP\// {status=$2} END {print status}' "$tile_headers")
+tile_content_type=$(
+  awk '
+    tolower($1) == "content-type:" {
+      value=$2
+      sub(/\r$/, "", value)
+    }
+    END {print value}
+  ' "$tile_headers"
+)
+tile_magic=$(od -An -tx1 -N8 "$tile_body" | tr -d ' \n')
+
+if [[ "$tile_status" != 200 ]] ||
+  [[ "$tile_content_type" != image/png ]] ||
+  [[ "$tile_magic" != 89504e470d0a1a0a ]]; then
+  echo "The canonical OSM boundary probe did not return a valid PNG tile." >&2
+  exit 1
+fi
+
+jq -n \
+  --arg url "https://tile.openstreetmap.org/0/0/0.png" \
+  --arg status "$tile_status" \
+  --arg content_type "$tile_content_type" \
+  --arg sha256 "$(sha256sum "$tile_body" | awk '{print $1}')" \
+  '{url: $url, status: ($status | tonumber), content_type: $content_type, sha256: $sha256}' \
+  >"$output_dir/osm-tile-boundary.json"
+
 git archive "$commit" | tar -x -C "$archive_dir"
 # The deployed application remains the exact production commit. Only the
 # run-scoped fixture task and browser assertion are overlaid into throwaway
@@ -394,6 +432,7 @@ docker run --rm \
   --env "E2E_FIXTURE_PASSWORD=$fixture_password" \
   --env "E2E_ADMIN_EMAIL=wnh-staging-e2e-$RUN_ID-admin@example.invalid" \
   --env E2E_PRODUCTION_READ_ONLY=1 \
+  --env E2E_DETERMINISTIC_MAP_TILES=1 \
   --env HOME=/tmp \
   --volume "$output_dir/browser:/work/output" \
   "$browser_image" \