feat: add verified operations tooling

.env.example

@@ -36,6 +36,7 @@ POOL_SIZE=10
 SECRET_KEY_BASE=generate-with-mix-phx-gen-secret
 HANDOVER_SECRET=generate-an-independent-random-secret
 RELEASE_COOKIE=generate-an-independent-beam-cluster-cookie
+METRICS_TOKEN=generate-an-independent-random-bearer-token
 
 # Mailpit settings for local development. Replace these with the selected SMTP
 # provider for public registration and magic links.

Dockerfile

@@ -85,7 +85,7 @@ RUN mix release
 FROM ${RUNNER_IMAGE} AS final
 
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates curl \
+  && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 libsctp1 locales ca-certificates curl \
   && rm -rf /var/lib/apt/lists/*
 
 # Set the locale

README.md

@@ -84,6 +84,17 @@ docker compose -p who_need_help logs -f web worker
 ./scripts/verify-realtime-cluster.sh compose
 ```
 
+Create and restore-test a database backup without restoring over the source:
+
+```bash
+backup_output=$(./scripts/backup-compose.sh)
+backup_path=$(printf '%s\n' "$backup_output" | sed -n 's/^Backup: //p')
+./scripts/restore-drill-compose.sh "$backup_path"
+```
+
+The commands, boundaries, and unclaimed production properties are documented
+in [the operations runbook](docs/operations.md).
+
 Local defaults are intentionally limited to local development. Copy
 `.env.example` to `.env` and replace every secret before any public deployment.
 Generate independent values with `mix phx.gen.secret`; do not reuse the session
@@ -189,10 +200,10 @@ credential fields.
 For an external cluster, provide a real PostgreSQL/PostGIS service and a
 pre-created Secret through required `existingSecret`; the chart never renders
 credentials from tracked values. The Secret must contain `DATABASE_URL`,
-`SECRET_KEY_BASE`, `HANDOVER_SECRET`, and `RELEASE_COOKIE`, and may additionally
-contain both GitHub OAuth variables described above. The chart intentionally
-has no invented CPU/RAM limits or HPA thresholds; measure this application in
-the target environment before setting them.
+`SECRET_KEY_BASE`, `HANDOVER_SECRET`, `RELEASE_COOKIE`, and `METRICS_TOKEN`, and
+may additionally contain both GitHub OAuth variables described above. The chart
+intentionally has no invented CPU/RAM limits or HPA thresholds; measure this
+application in the target environment before setting them.
 
 ## Local Codex category review
 
@@ -220,6 +231,7 @@ and no fallback provider. Recommendations require a human moderator action.
 - [Product specification](docs/product-spec.md)
 - [Architecture](docs/architecture.md)
 - [Trust and safety](docs/trust-safety.md)
+- [Operations runbook](docs/operations.md)
 - [Implementation verification and known limits](docs/verification.md)
 - [Verified dependency baseline](docs/dependency-baseline.md)
 - [PostgreSQL/PostGIS ADR](docs/decisions/0001-postgresql-postgis-over-spacetimedb.md)

compose.yaml

@@ -6,6 +6,7 @@ x-app-environment: &app-environment
   SECRET_KEY_BASE: ${SECRET_KEY_BASE:?Set SECRET_KEY_BASE in .env}
   HANDOVER_SECRET: ${HANDOVER_SECRET:?Set HANDOVER_SECRET in .env}
   RELEASE_COOKIE: ${RELEASE_COOKIE:?Set RELEASE_COOKIE in .env}
+  METRICS_TOKEN: ${METRICS_TOKEN:?Set an independent METRICS_TOKEN in .env}
   DNS_CLUSTER_QUERY: web
   PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env}
   PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env}

config/runtime.exs

@@ -49,6 +49,21 @@ github_oauth =
 
 config :who_need_help, :social_oauth, github_oauth
 
+if config_env() == :prod and app_role == :web do
+  metrics_token =
+    System.get_env("METRICS_TOKEN") ||
+      raise """
+      environment variable METRICS_TOKEN is missing for the web role.
+      Generate an independent random value and store it in the deployment secret.
+      """
+
+  if metrics_token == "" do
+    raise "METRICS_TOKEN must not be empty for the web role."
+  end
+
+  config :who_need_help, :metrics_token, metrics_token
+end
+
 if dns_query = System.get_env("DNS_CLUSTER_QUERY") do
   config :who_need_help, :dns_cluster_query, dns_query
 end

config/test.exs

@@ -27,6 +27,7 @@ config :who_need_help, WhoNeedHelpWeb.Endpoint,
 # In test we don't send emails
 config :who_need_help, WhoNeedHelp.Mailer, adapter: Swoosh.Adapters.Test
 config :who_need_help, :social_oauth_adapter, WhoNeedHelp.SocialOAuthFake
+config :who_need_help, :metrics_token, "test-metrics-token"
 
 # Disable swoosh api client as it is only required for production adapters
 config :swoosh, :api_client, false

deploy/helm/who-need-help/values.yaml

@@ -28,7 +28,7 @@ app:
   smtpPort: "1025"
 
 # Required. The Secret must contain DATABASE_URL, SECRET_KEY_BASE,
-# HANDOVER_SECRET, and RELEASE_COOKIE. It may also contain both
+# HANDOVER_SECRET, RELEASE_COOKIE, and METRICS_TOKEN. It may also contain both
 # GITHUB_OAUTH_CLIENT_ID and GITHUB_OAUTH_CLIENT_SECRET to enable verified
 # GitHub linking.
 existingSecret: ""

docs/operations.md

@@ -0,0 +1,111 @@
+# Operations runbook
+
+This runbook describes the commands that are implemented and verified in this
+repository. It does not claim a production recovery point objective, recovery
+time objective, retention period, storage capacity, or high-availability model;
+those values require product policy and measurements from the eventual
+production environment.
+
+## Compose database backup
+
+Create a PostgreSQL 18 custom-format archive, validate its table of contents,
+and write a SHA-256 manifest:
+
+```bash
+./scripts/backup-compose.sh
+```
+
+The default destination is the ignored `output/backups/` directory. An explicit
+new destination may be supplied as the only argument. The command refuses to
+overwrite either an archive or its checksum manifest and writes through
+temporary files before publishing the final pair. Files and newly created
+directories are restricted by `umask 077`.
+
+The archive covers the configured application database. PostgreSQL cluster
+globals such as roles and tablespaces are not part of `pg_dump`; deployment
+credentials and database roles must be provisioned separately from secrets.
+Local backup files on the same workstation are not an off-site backup.
+
+## Isolated restore drill
+
+Run a real restore into a uniquely named temporary database:
+
+```bash
+./scripts/restore-drill-compose.sh output/backups/compose-YYYYMMDD-HHMMSS.dump
+```
+
+The drill:
+
+1. validates the SHA-256 manifest;
+2. validates the archive table of contents;
+3. creates a pristine database from `template0`;
+4. restores with `pg_restore --exit-on-error`;
+5. reads every restored public application table and checks the PostGIS
+   library;
+6. runs the current immutable release's migrations and migration-readiness check
+   against only the temporary database;
+7. drops only the temporary drill database and verifies that it is gone.
+
+A trap also attempts to drop the exact temporary database if a check fails.
+The source application database is never passed to `pg_restore`, `dropdb`, a
+clean operation, or the drill migration runner. The drill intentionally does
+not compare an older backup's row counts to the live source, because concurrent
+legitimate writes or a historical archive would make that comparison invalid.
+
+## Service checks
+
+```bash
+docker compose ps
+curl --fail http://localhost:4010/healthz/live
+curl --fail http://localhost:4010/healthz/ready
+./scripts/verify-realtime-cluster.sh compose
+```
+
+`live` verifies that the web process can serve HTTP. `ready` additionally runs
+`SELECT 1` through the configured Ecto repository. The cluster probe subscribes
+on one connected BEAM node and broadcasts from another.
+
+Health checks do not replace alerting, database backups, restore drills, or
+application-level synthetic checks.
+
+## Protected Prometheus metrics
+
+The web role exposes Prometheus text format at `/metrics`. It requires the
+independent `METRICS_TOKEN` deployment secret:
+
+```bash
+curl --fail \
+  --header "Authorization: Bearer $METRICS_TOKEN" \
+  http://localhost:4010/metrics
+```
+
+The endpoint returns `401` without the exact token, disables response caching,
+and does not put the credential in a URL. The reporter exports cumulative HTTP
+request and duration, router exception, database query and duration, WebSocket
+connection, VM memory, and scheduler run-queue metrics. Cumulative durations are
+integer microseconds because the selected reporter's sum accumulator is
+integer-based; divide by `1_000_000` in PromQL when seconds are required.
+Definitions intentionally have no request path, user, request, or event-name
+labels that could create unbounded cardinality.
+
+Prometheus itself, durable metrics retention, alert rules, notification
+destinations, and measured alert thresholds are deployment responsibilities and
+are not claimed by this repository. In Kubernetes, put the token in
+`existingSecret`; configure the external scraper to send it as a Bearer token.
+
+Metrics are local to each BEAM process. Discover and scrape every web pod or
+container as a distinct target and preserve Prometheus's `instance` label. A
+request through the load-balanced public route reaches only one replica and is
+therefore useful as an authorization/smoke check, not as a cluster-wide
+aggregate.
+
+## Rollback boundary
+
+The release image is immutable and migrations run as a separate one-shot role.
+Before a schema rollout, create and restore-test a current backup. Application
+rollback and database migration rollback are separate decisions: do not run an
+Ecto down migration merely because an image is rolled back. Inspect the exact
+migration and compatibility boundary first.
+
+The repository intentionally does not ship an automatic destructive production
+restore command.

docs/verification.md

@@ -174,6 +174,32 @@ Local ignored browser evidence:
 
 - `.playwright-cli/page-2026-07-18T18-09-44-303Z.png`
 
+## Operations and metrics verification
+
+On 2026-07-18, the Compose database was archived with PostgreSQL 18
+custom-format `pg_dump`. The published archive and SHA-256 sidecar were
+validated before an isolated restore. The restore drill created a database from
+`template0`, restored with `pg_restore --exit-on-error`, read 23 public
+application tables and 1106 restored rows, reported PostGIS 3.6.4, observed all
+8 current migrations, and removed the exact temporary database. A follow-up
+catalog query returned zero remaining restore-drill databases. Source counts
+before and after remained `2 users / 1 help request / 7 messages / 14 categories
+/ 1 assignment / 0 activities / 0 reports / 0 social identities`.
+
+The same rollout started 2 healthy web replicas and 2 worker replicas. Local
+live/readiness, public HTTPS readiness, and the public root returned HTTP 200.
+The cross-node PubSub probe passed across all four connected BEAM nodes. The
+protected metrics route returned HTTP 401 both locally and through public HTTPS
+without credentials, returned valid Prometheus text with the generated ignored
+local token, and was scraped directly from each web container. Helm 4.2.3 lint
+and template rendering passed.
+
+Before the runtime image change, every BEAM container logged that
+`libsctp.so.1` was unavailable. The rebuilt Debian trixie release contains the
+`libsctp1` package and the exact shared library; no SCTP, application error, or
+application warning appeared in the post-rollout web/worker logs checked during
+this verification window.
+
 ## Known work before a public production launch
 
 - Replace the temporary staging origin with the production-owned domain and

lib/who_need_help_web/controllers/metrics_controller.ex

@@ -0,0 +1,31 @@
+defmodule WhoNeedHelpWeb.MetricsController do
+  use WhoNeedHelpWeb, :controller
+
+  @content_type "text/plain; version=0.0.4"
+
+  def show(conn, _params) do
+    token = Application.get_env(:who_need_help, :metrics_token)
+
+    if authorized?(get_req_header(conn, "authorization"), token) do
+      body = TelemetryMetricsPrometheus.Core.scrape(:prometheus_metrics)
+
+      conn
+      |> put_resp_header("cache-control", "no-store")
+      |> put_resp_content_type(@content_type)
+      |> send_resp(:ok, body)
+    else
+      conn
+      |> put_resp_header("cache-control", "no-store")
+      |> put_resp_header("www-authenticate", "Bearer")
+      |> send_resp(:unauthorized, "Unauthorized\n")
+    end
+  end
+
+  defp authorized?(["Bearer " <> candidate], expected)
+       when is_binary(expected) and expected != "" and
+              byte_size(candidate) == byte_size(expected) do
+    Plug.Crypto.secure_compare(candidate, expected)
+  end
+
+  defp authorized?(_authorization, _expected), do: false
+end

lib/who_need_help_web/router.ex

@@ -33,6 +33,10 @@ defmodule WhoNeedHelpWeb.Router do
     get "/ready", HealthController, :ready
   end
 
+  scope "/", WhoNeedHelpWeb do
+    get "/metrics", MetricsController, :show
+  end
+
   scope "/mobile", WhoNeedHelpWeb do
     pipe_through :mobile
 

lib/who_need_help_web/telemetry.ex

@@ -8,17 +8,87 @@ defmodule WhoNeedHelpWeb.Telemetry do
 
   @impl true
   def init(_arg) do
-    children = [
-      # Telemetry poller will execute the given period measurements
-      # every 10_000ms. Learn more here: https://telemetry-metrics.hexdocs.pm
-      {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}
-      # Add reporters as children of your supervision tree.
-      # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}
-    ]
+    reporter_children =
+      if Application.fetch_env!(:who_need_help, :app_role) == :web do
+        [
+          {TelemetryMetricsPrometheus.Core,
+           name: :prometheus_metrics, metrics: prometheus_metrics(), start_async: false}
+        ]
+      else
+        []
+      end
+
+    children =
+      reporter_children ++
+        [
+          # Telemetry poller will execute the given period measurements
+          # every 10_000ms. Learn more here: https://telemetry-metrics.hexdocs.pm
+          {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}
+        ]
 
     Supervisor.init(children, strategy: :one_for_one)
   end
 
+  def prometheus_metrics do
+    [
+      counter("who_need_help.http.requests.total",
+        event_name: [:phoenix, :endpoint, :stop],
+        measurement: :duration,
+        description: "Completed HTTP requests"
+      ),
+      sum("who_need_help.http.request.duration.microseconds.total",
+        event_name: [:phoenix, :endpoint, :stop],
+        measurement: fn measurements ->
+          System.convert_time_unit(measurements.duration, :native, :microsecond)
+        end,
+        description: "Cumulative HTTP request duration"
+      ),
+      counter("who_need_help.http.exceptions.total",
+        event_name: [:phoenix, :router_dispatch, :exception],
+        measurement: :duration,
+        description: "Router dispatch exceptions"
+      ),
+      counter("who_need_help.database.queries.total",
+        event_name: [:who_need_help, :repo, :query],
+        measurement: :total_time,
+        description: "Completed database queries"
+      ),
+      sum("who_need_help.database.query.duration.microseconds.total",
+        event_name: [:who_need_help, :repo, :query],
+        measurement: fn measurements ->
+          System.convert_time_unit(measurements.total_time, :native, :microsecond)
+        end,
+        description: "Cumulative database query duration"
+      ),
+      counter("who_need_help.websocket.connections.total",
+        event_name: [:phoenix, :socket_connected],
+        measurement: :duration,
+        description: "Completed WebSocket connection attempts"
+      ),
+      last_value("who_need_help.vm.memory.total.bytes",
+        event_name: [:vm, :memory],
+        measurement: :total,
+        unit: :byte,
+        description: "Total memory used by the Erlang VM"
+      ),
+      last_value("who_need_help.vm.run_queue.total",
+        event_name: [:vm, :total_run_queue_lengths],
+        measurement: :total,
+        description: "Total scheduler run queue length"
+      ),
+      last_value("who_need_help.vm.run_queue.cpu",
+        event_name: [:vm, :total_run_queue_lengths],
+        measurement: :cpu,
+        description: "CPU scheduler run queue length"
+      ),
+      last_value("who_need_help.vm.run_queue.io",
+        event_name: [:vm, :total_run_queue_lengths],
+        measurement: :io,
+        description: "I/O scheduler run queue length"
+      )
+    ]
+  end
+
   def metrics do
     [
       # Phoenix Metrics

mix.exs

@@ -73,6 +73,7 @@ defmodule WhoNeedHelp.MixProject do
       {:req, "~> 0.5"},
       {:assent, "~> 0.3.1"},
       {:telemetry_metrics, "~> 1.0"},
+      {:telemetry_metrics_prometheus_core, "~> 1.2.1"},
       {:telemetry_poller, "~> 1.0"},
       {:gettext, "~> 1.0"},
       {:jason, "~> 1.2"},

mix.lock

@@ -47,6 +47,7 @@
   "tailwind": {:hex, :tailwind, "0.5.1", "35435b13158c90d37da11e1cfc808755fca1d7b6c5ab87b1b19c5de87e2f0a10", [:mix], [], "hexpm", "c4e26302a59fec72abc5610ecb6ad2116d9aa31f31aab2d4b8eb6e95d25a689c"},
   "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
+  "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},
   "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},

scripts/backup-compose.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+set -eu
+umask 077
+
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+cd "$ROOT"
+
+if [ ! -f "$ROOT/.env" ]; then
+  echo "Missing $ROOT/.env." >&2
+  exit 1
+fi
+
+set -a
+. "$ROOT/.env"
+set +a
+
+: "${POSTGRES_DB:?Set POSTGRES_DB in .env}"
+: "${POSTGRES_USER:?Set POSTGRES_USER in .env}"
+
+timestamp=$(date -u +%Y%m%d-%H%M%S)
+target=${1:-"$ROOT/output/backups/compose-$timestamp.dump"}
+target_dir=$(dirname -- "$target")
+target_name=$(basename -- "$target")
+partial="$target.partial.$$"
+checksum="$target.sha256"
+checksum_partial="$checksum.partial.$$"
+
+if [ -e "$target" ] || [ -e "$checksum" ]; then
+  echo "Refusing to overwrite $target or $checksum." >&2
+  exit 1
+fi
+
+mkdir -p "$target_dir"
+
+cleanup() {
+  rm -f "$partial" "$checksum_partial"
+}
+trap cleanup EXIT HUP INT TERM
+
+docker compose exec -T db \
+  pg_dump \
+  --username "$POSTGRES_USER" \
+  --dbname "$POSTGRES_DB" \
+  --format custom \
+  --no-owner >"$partial"
+
+test -s "$partial"
+
+docker compose exec -T db pg_restore --list <"$partial" >/dev/null
+
+hash=$(sha256sum "$partial" | awk '{print $1}')
+printf '%s  %s\n' "$hash" "$target_name" >"$checksum_partial"
+
+mv "$partial" "$target"
+mv "$checksum_partial" "$checksum"
+trap - EXIT HUP INT TERM
+
+echo "Backup: $target"
+echo "Checksum: $checksum"
+echo "SHA-256: $hash"

scripts/ensure-local-metrics-token.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+set -eu
+
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+ENV_FILE="$ROOT/.env"
+
+if [ ! -f "$ENV_FILE" ]; then
+  echo "Missing $ENV_FILE. Copy .env.example to .env first." >&2
+  exit 1
+fi
+
+for command in openssl perl; do
+  if ! command -v "$command" >/dev/null 2>&1; then
+    echo "Required command is unavailable: $command" >&2
+    exit 1
+  fi
+done
+
+current=$(
+  perl -ne '
+    if (/^METRICS_TOKEN=(.*)$/) {
+      print $1;
+      exit;
+    }
+  ' "$ENV_FILE"
+)
+
+if [ -n "$current" ]; then
+  chmod 600 "$ENV_FILE"
+  echo "METRICS_TOKEN already exists; no secret was changed."
+  exit 0
+fi
+
+new_metrics_token=$(openssl rand -hex 32)
+tmp_env=$(mktemp "${ENV_FILE}.metrics.XXXXXX")
+trap 'rm -f "$tmp_env"' EXIT HUP INT TERM
+chmod 600 "$tmp_env"
+
+NEW_METRICS_TOKEN=$new_metrics_token perl -0pe '
+  if (s/^METRICS_TOKEN=.*$/METRICS_TOKEN=$ENV{NEW_METRICS_TOKEN}/m) {
+    $replaced = 1;
+  }
+
+  END {
+    print "\nMETRICS_TOKEN=$ENV{NEW_METRICS_TOKEN}\n" unless $replaced;
+  }
+' "$ENV_FILE" >"$tmp_env"
+
+mv "$tmp_env" "$ENV_FILE"
+chmod 600 "$ENV_FILE"
+trap - EXIT HUP INT TERM
+unset new_metrics_token
+
+echo "Generated METRICS_TOKEN in the ignored .env without printing it."

scripts/kind-up.sh

@@ -108,6 +108,7 @@ if ! kube --namespace "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; th
   secret_key_base=$(openssl rand -hex 64)
   handover_secret=$(openssl rand -hex 64)
   release_cookie=$(openssl rand -hex 64)
+  metrics_token=$(openssl rand -hex 32)
 
   kube --namespace "$NAMESPACE" create secret generic "$SECRET_NAME" \
     --from-literal=POSTGRES_DB=who_need_help \
@@ -116,9 +117,18 @@ if ! kube --namespace "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; th
     --from-literal="DATABASE_URL=ecto://postgres:${postgres_password}@postgis/who_need_help" \
     --from-literal="SECRET_KEY_BASE=$secret_key_base" \
     --from-literal="HANDOVER_SECRET=$handover_secret" \
-    --from-literal="RELEASE_COOKIE=$release_cookie"
+    --from-literal="RELEASE_COOKIE=$release_cookie" \
+    --from-literal="METRICS_TOKEN=$metrics_token"
 
-  unset postgres_password secret_key_base handover_secret release_cookie
+  unset postgres_password secret_key_base handover_secret release_cookie metrics_token
+elif [ -z "$(kube --namespace "$NAMESPACE" get secret "$SECRET_NAME" -o jsonpath='{.data.METRICS_TOKEN}')" ]; then
+  metrics_token=$(openssl rand -hex 32)
+
+  kube --namespace "$NAMESPACE" patch secret "$SECRET_NAME" \
+    --type merge \
+    --patch "{\"stringData\":{\"METRICS_TOKEN\":\"$metrics_token\"}}" >/dev/null
+
+  unset metrics_token
 fi
 
 if [ -n "$legacy_backup" ]; then

scripts/restore-drill-compose.sh

@@ -0,0 +1,169 @@
+#!/bin/sh
+set -eu
+
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+cd "$ROOT"
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 PATH_TO_CUSTOM_FORMAT_DUMP" >&2
+  exit 1
+fi
+
+dump=$1
+checksum="$dump.sha256"
+
+if [ ! -f "$dump" ] || [ ! -s "$dump" ]; then
+  echo "Dump does not exist or is empty: $dump" >&2
+  exit 1
+fi
+
+if [ ! -f "$checksum" ]; then
+  echo "Checksum manifest is required: $checksum" >&2
+  exit 1
+fi
+
+if [ ! -f "$ROOT/.env" ]; then
+  echo "Missing $ROOT/.env." >&2
+  exit 1
+fi
+
+set -a
+. "$ROOT/.env"
+set +a
+
+: "${POSTGRES_DB:?Set POSTGRES_DB in .env}"
+: "${POSTGRES_USER:?Set POSTGRES_USER in .env}"
+: "${DATABASE_URL:?Set DATABASE_URL in .env}"
+
+dump_dir=$(CDPATH= cd -- "$(dirname -- "$dump")" && pwd)
+dump_name=$(basename -- "$dump")
+
+(
+  cd "$dump_dir"
+  sha256sum --check --status "$(basename -- "$checksum")"
+)
+
+docker compose exec -T db pg_restore --list <"$dump" >/dev/null
+
+drill_db="who_need_help_restore_drill_$(date -u +%Y%m%d%H%M%S)_$$"
+created=false
+
+cleanup() {
+  if [ "$created" = true ]; then
+    docker compose exec -T db \
+      dropdb --username "$POSTGRES_USER" --if-exists "$drill_db" >/dev/null
+  fi
+}
+trap cleanup EXIT HUP INT TERM
+
+docker compose exec -T db \
+  createdb \
+  --username "$POSTGRES_USER" \
+  --template template0 \
+  "$drill_db"
+created=true
+
+docker compose exec -T db \
+  pg_restore \
+  --username "$POSTGRES_USER" \
+  --dbname "$drill_db" \
+  --exit-on-error \
+  --no-owner \
+  --no-privileges <"$dump"
+
+tables=$(
+  docker compose exec -T db \
+    psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+    --command "
+      SELECT tablename
+      FROM pg_tables
+      WHERE schemaname = 'public'
+        AND tablename <> 'spatial_ref_sys'
+      ORDER BY tablename;
+    "
+)
+
+table_count=0
+total_rows=0
+
+for table in $tables; do
+  restored_count=$(
+    docker compose exec -T db \
+      psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+      --command "SELECT count(*) FROM \"$table\";"
+  )
+
+  table_count=$((table_count + 1))
+  total_rows=$((total_rows + restored_count))
+done
+
+if [ "$table_count" -eq 0 ]; then
+  echo "The restored database contains no public application tables." >&2
+  exit 1
+fi
+
+postgis_version=$(
+  docker compose exec -T db \
+    psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+    --command "SELECT PostGIS_Lib_Version();"
+)
+
+case "$DATABASE_URL" in
+  *\?*)
+    database_base=${DATABASE_URL%%\?*}
+    database_query="?${DATABASE_URL#*\?}"
+    ;;
+
+  *)
+    database_base=$DATABASE_URL
+    database_query=
+    ;;
+esac
+
+database_prefix=${database_base%/*}
+
+if [ "$database_prefix" = "$database_base" ]; then
+  echo "Could not derive the temporary drill DATABASE_URL." >&2
+  exit 1
+fi
+
+drill_database_url="$database_prefix/$drill_db$database_query"
+
+docker compose run --rm --no-deps \
+  --env APP_ROLE=migrate \
+  --env "DATABASE_URL=$drill_database_url" \
+  migrate /app/bin/migrate
+
+docker compose run --rm --no-deps \
+  --env APP_ROLE=migrate \
+  --env "DATABASE_URL=$drill_database_url" \
+  migrate /app/bin/who_need_help eval 'WhoNeedHelp.Release.await_migrations()'
+
+migration_count=$(
+  docker compose exec -T db \
+    psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \
+    --command "SELECT count(*) FROM schema_migrations;"
+)
+
+docker compose exec -T db \
+  dropdb --username "$POSTGRES_USER" "$drill_db"
+created=false
+trap - EXIT HUP INT TERM
+
+remaining=$(
+  docker compose exec -T db \
+    psql --username "$POSTGRES_USER" --dbname postgres --tuples-only --no-align \
+    --command "SELECT count(*) FROM pg_database WHERE datname = '$drill_db';"
+)
+
+if [ "$remaining" != "0" ]; then
+  echo "Restore drill database was not removed: $drill_db" >&2
+  exit 1
+fi
+
+echo "Restore drill passed for: $dump_name"
+echo "Readable public tables: $table_count"
+echo "Restored rows read: $total_rows"
+echo "Applied migrations: $migration_count"
+echo "PostGIS library: $postgis_version"
+echo "Temporary database removed: $drill_db"

test/who_need_help_web/controllers/metrics_controller_test.exs

@@ -0,0 +1,40 @@
+defmodule WhoNeedHelpWeb.MetricsControllerTest do
+  use WhoNeedHelpWeb.ConnCase, async: false
+
+  test "rejects a request without credentials", %{conn: conn} do
+    conn = get(conn, "/metrics")
+
+    assert response(conn, 401) == "Unauthorized\n"
+    assert get_resp_header(conn, "www-authenticate") == ["Bearer"]
+    assert get_resp_header(conn, "cache-control") == ["no-store"]
+  end
+
+  test "rejects an incorrect bearer token", %{conn: conn} do
+    conn =
+      conn
+      |> put_req_header("authorization", "Bearer incorrect-token")
+      |> get("/metrics")
+
+    assert response(conn, 401) == "Unauthorized\n"
+  end
+
+  test "exports Prometheus metrics for the configured bearer token", %{conn: conn} do
+    get(conn, "/healthz/live")
+
+    conn =
+      conn
+      |> put_req_header("authorization", "Bearer test-metrics-token")
+      |> get("/metrics")
+
+    body = response(conn, 200)
+
+    assert get_resp_header(conn, "content-type") == [
+             "text/plain; version=0.0.4; charset=utf-8"
+           ]
+
+    assert get_resp_header(conn, "cache-control") == ["no-store"]
+    assert body =~ "# TYPE who_need_help_http_requests_total counter"
+    assert body =~ "who_need_help_http_requests_total "
+    assert body =~ "who_need_help_http_request_duration_microseconds_total "
+  end
+end