From 4d5efea41946234b8fe208a08e0a0215f92a8a7e Mon Sep 17 00:00:00 2001 From: SimpleTest Date: Sat, 18 Jul 2026 21:34:31 +0300 Subject: [PATCH] feat: add verified operations tooling --- .env.example | 1 + Dockerfile | 2 +- README.md | 20 ++- compose.yaml | 1 + config/runtime.exs | 15 ++ config/test.exs | 1 + deploy/helm/who-need-help/values.yaml | 2 +- docs/operations.md | 111 ++++++++++++ docs/verification.md | 26 +++ .../controllers/metrics_controller.ex | 31 ++++ lib/who_need_help_web/router.ex | 4 + lib/who_need_help_web/telemetry.ex | 84 ++++++++- mix.exs | 1 + mix.lock | 1 + scripts/backup-compose.sh | 60 +++++++ scripts/ensure-local-metrics-token.sh | 54 ++++++ scripts/kind-up.sh | 14 +- scripts/restore-drill-compose.sh | 169 ++++++++++++++++++ .../controllers/metrics_controller_test.exs | 40 +++++ 19 files changed, 622 insertions(+), 15 deletions(-) create mode 100644 docs/operations.md create mode 100644 lib/who_need_help_web/controllers/metrics_controller.ex create mode 100755 scripts/backup-compose.sh create mode 100755 scripts/ensure-local-metrics-token.sh create mode 100755 scripts/restore-drill-compose.sh create mode 100644 test/who_need_help_web/controllers/metrics_controller_test.exs diff --git a/.env.example b/.env.example index 723aad8..5ec9c55 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,7 @@ POOL_SIZE=10 SECRET_KEY_BASE=generate-with-mix-phx-gen-secret HANDOVER_SECRET=generate-an-independent-random-secret RELEASE_COOKIE=generate-an-independent-beam-cluster-cookie +METRICS_TOKEN=generate-an-independent-random-bearer-token # Mailpit settings for local development. Replace these with the selected SMTP # provider for public registration and magic links. diff --git a/Dockerfile b/Dockerfile index 0cee593..aaf4265 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,7 +85,7 @@ RUN mix release FROM ${RUNNER_IMAGE} AS final RUN apt-get update \ - && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates curl \ + && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 libsctp1 locales ca-certificates curl \ && rm -rf /var/lib/apt/lists/* # Set the locale diff --git a/README.md b/README.md index 7d681aa..79a35e3 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,17 @@ docker compose -p who_need_help logs -f web worker ./scripts/verify-realtime-cluster.sh compose ``` +Create and restore-test a database backup without restoring over the source: + +```bash +backup_output=$(./scripts/backup-compose.sh) +backup_path=$(printf '%s\n' "$backup_output" | sed -n 's/^Backup: //p') +./scripts/restore-drill-compose.sh "$backup_path" +``` + +The commands, boundaries, and unclaimed production properties are documented +in [the operations runbook](docs/operations.md). + Local defaults are intentionally limited to local development. Copy `.env.example` to `.env` and replace every secret before any public deployment. Generate independent values with `mix phx.gen.secret`; do not reuse the session @@ -189,10 +200,10 @@ credential fields. For an external cluster, provide a real PostgreSQL/PostGIS service and a pre-created Secret through required `existingSecret`; the chart never renders credentials from tracked values. The Secret must contain `DATABASE_URL`, -`SECRET_KEY_BASE`, `HANDOVER_SECRET`, and `RELEASE_COOKIE`, and may additionally -contain both GitHub OAuth variables described above. The chart intentionally -has no invented CPU/RAM limits or HPA thresholds; measure this application in -the target environment before setting them. +`SECRET_KEY_BASE`, `HANDOVER_SECRET`, `RELEASE_COOKIE`, and `METRICS_TOKEN`, and +may additionally contain both GitHub OAuth variables described above. The chart +intentionally has no invented CPU/RAM limits or HPA thresholds; measure this +application in the target environment before setting them. ## Local Codex category review @@ -220,6 +231,7 @@ and no fallback provider. Recommendations require a human moderator action. - [Product specification](docs/product-spec.md) - [Architecture](docs/architecture.md) - [Trust and safety](docs/trust-safety.md) +- [Operations runbook](docs/operations.md) - [Implementation verification and known limits](docs/verification.md) - [Verified dependency baseline](docs/dependency-baseline.md) - [PostgreSQL/PostGIS ADR](docs/decisions/0001-postgresql-postgis-over-spacetimedb.md) diff --git a/compose.yaml b/compose.yaml index a935d9d..2e05505 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,7 @@ x-app-environment: &app-environment SECRET_KEY_BASE: ${SECRET_KEY_BASE:?Set SECRET_KEY_BASE in .env} HANDOVER_SECRET: ${HANDOVER_SECRET:?Set HANDOVER_SECRET in .env} RELEASE_COOKIE: ${RELEASE_COOKIE:?Set RELEASE_COOKIE in .env} + METRICS_TOKEN: ${METRICS_TOKEN:?Set an independent METRICS_TOKEN in .env} DNS_CLUSTER_QUERY: web PHX_HOST: ${PHX_HOST:?Set PHX_HOST in .env} PHX_SCHEME: ${PHX_SCHEME:?Set PHX_SCHEME in .env} diff --git a/config/runtime.exs b/config/runtime.exs index 9980fa3..d007d77 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -49,6 +49,21 @@ github_oauth = config :who_need_help, :social_oauth, github_oauth +if config_env() == :prod and app_role == :web do + metrics_token = + System.get_env("METRICS_TOKEN") || + raise """ + environment variable METRICS_TOKEN is missing for the web role. + Generate an independent random value and store it in the deployment secret. + """ + + if metrics_token == "" do + raise "METRICS_TOKEN must not be empty for the web role." + end + + config :who_need_help, :metrics_token, metrics_token +end + if dns_query = System.get_env("DNS_CLUSTER_QUERY") do config :who_need_help, :dns_cluster_query, dns_query end diff --git a/config/test.exs b/config/test.exs index c9ccfe4..3c7d2f4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -27,6 +27,7 @@ config :who_need_help, WhoNeedHelpWeb.Endpoint, # In test we don't send emails config :who_need_help, WhoNeedHelp.Mailer, adapter: Swoosh.Adapters.Test config :who_need_help, :social_oauth_adapter, WhoNeedHelp.SocialOAuthFake +config :who_need_help, :metrics_token, "test-metrics-token" # Disable swoosh api client as it is only required for production adapters config :swoosh, :api_client, false diff --git a/deploy/helm/who-need-help/values.yaml b/deploy/helm/who-need-help/values.yaml index 83b1cf5..b621dd7 100644 --- a/deploy/helm/who-need-help/values.yaml +++ b/deploy/helm/who-need-help/values.yaml @@ -28,7 +28,7 @@ app: smtpPort: "1025" # Required. The Secret must contain DATABASE_URL, SECRET_KEY_BASE, -# HANDOVER_SECRET, and RELEASE_COOKIE. It may also contain both +# HANDOVER_SECRET, RELEASE_COOKIE, and METRICS_TOKEN. It may also contain both # GITHUB_OAUTH_CLIENT_ID and GITHUB_OAUTH_CLIENT_SECRET to enable verified # GitHub linking. existingSecret: "" diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..21bcc01 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,111 @@ +# Operations runbook + +This runbook describes the commands that are implemented and verified in this +repository. It does not claim a production recovery point objective, recovery +time objective, retention period, storage capacity, or high-availability model; +those values require product policy and measurements from the eventual +production environment. + +## Compose database backup + +Create a PostgreSQL 18 custom-format archive, validate its table of contents, +and write a SHA-256 manifest: + +```bash +./scripts/backup-compose.sh +``` + +The default destination is the ignored `output/backups/` directory. An explicit +new destination may be supplied as the only argument. The command refuses to +overwrite either an archive or its checksum manifest and writes through +temporary files before publishing the final pair. Files and newly created +directories are restricted by `umask 077`. + +The archive covers the configured application database. PostgreSQL cluster +globals such as roles and tablespaces are not part of `pg_dump`; deployment +credentials and database roles must be provisioned separately from secrets. +Local backup files on the same workstation are not an off-site backup. + +## Isolated restore drill + +Run a real restore into a uniquely named temporary database: + +```bash +./scripts/restore-drill-compose.sh output/backups/compose-YYYYMMDD-HHMMSS.dump +``` + +The drill: + +1. validates the SHA-256 manifest; +2. validates the archive table of contents; +3. creates a pristine database from `template0`; +4. restores with `pg_restore --exit-on-error`; +5. reads every restored public application table and checks the PostGIS + library; +6. runs the current immutable release's migrations and migration-readiness check + against only the temporary database; +7. drops only the temporary drill database and verifies that it is gone. + +A trap also attempts to drop the exact temporary database if a check fails. +The source application database is never passed to `pg_restore`, `dropdb`, a +clean operation, or the drill migration runner. The drill intentionally does +not compare an older backup's row counts to the live source, because concurrent +legitimate writes or a historical archive would make that comparison invalid. + +## Service checks + +```bash +docker compose ps +curl --fail http://localhost:4010/healthz/live +curl --fail http://localhost:4010/healthz/ready +./scripts/verify-realtime-cluster.sh compose +``` + +`live` verifies that the web process can serve HTTP. `ready` additionally runs +`SELECT 1` through the configured Ecto repository. The cluster probe subscribes +on one connected BEAM node and broadcasts from another. + +Health checks do not replace alerting, database backups, restore drills, or +application-level synthetic checks. + +## Protected Prometheus metrics + +The web role exposes Prometheus text format at `/metrics`. It requires the +independent `METRICS_TOKEN` deployment secret: + +```bash +curl --fail \ + --header "Authorization: Bearer $METRICS_TOKEN" \ + http://localhost:4010/metrics +``` + +The endpoint returns `401` without the exact token, disables response caching, +and does not put the credential in a URL. The reporter exports cumulative HTTP +request and duration, router exception, database query and duration, WebSocket +connection, VM memory, and scheduler run-queue metrics. Cumulative durations are +integer microseconds because the selected reporter's sum accumulator is +integer-based; divide by `1_000_000` in PromQL when seconds are required. +Definitions intentionally have no request path, user, request, or event-name +labels that could create unbounded cardinality. + +Prometheus itself, durable metrics retention, alert rules, notification +destinations, and measured alert thresholds are deployment responsibilities and +are not claimed by this repository. In Kubernetes, put the token in +`existingSecret`; configure the external scraper to send it as a Bearer token. + +Metrics are local to each BEAM process. Discover and scrape every web pod or +container as a distinct target and preserve Prometheus's `instance` label. A +request through the load-balanced public route reaches only one replica and is +therefore useful as an authorization/smoke check, not as a cluster-wide +aggregate. + +## Rollback boundary + +The release image is immutable and migrations run as a separate one-shot role. +Before a schema rollout, create and restore-test a current backup. Application +rollback and database migration rollback are separate decisions: do not run an +Ecto down migration merely because an image is rolled back. Inspect the exact +migration and compatibility boundary first. + +The repository intentionally does not ship an automatic destructive production +restore command. diff --git a/docs/verification.md b/docs/verification.md index 39fa34a..223c13d 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -174,6 +174,32 @@ Local ignored browser evidence: - `.playwright-cli/page-2026-07-18T18-09-44-303Z.png` +## Operations and metrics verification + +On 2026-07-18, the Compose database was archived with PostgreSQL 18 +custom-format `pg_dump`. The published archive and SHA-256 sidecar were +validated before an isolated restore. The restore drill created a database from +`template0`, restored with `pg_restore --exit-on-error`, read 23 public +application tables and 1106 restored rows, reported PostGIS 3.6.4, observed all +8 current migrations, and removed the exact temporary database. A follow-up +catalog query returned zero remaining restore-drill databases. Source counts +before and after remained `2 users / 1 help request / 7 messages / 14 categories +/ 1 assignment / 0 activities / 0 reports / 0 social identities`. + +The same rollout started 2 healthy web replicas and 2 worker replicas. Local +live/readiness, public HTTPS readiness, and the public root returned HTTP 200. +The cross-node PubSub probe passed across all four connected BEAM nodes. The +protected metrics route returned HTTP 401 both locally and through public HTTPS +without credentials, returned valid Prometheus text with the generated ignored +local token, and was scraped directly from each web container. Helm 4.2.3 lint +and template rendering passed. + +Before the runtime image change, every BEAM container logged that +`libsctp.so.1` was unavailable. The rebuilt Debian trixie release contains the +`libsctp1` package and the exact shared library; no SCTP, application error, or +application warning appeared in the post-rollout web/worker logs checked during +this verification window. + ## Known work before a public production launch - Replace the temporary staging origin with the production-owned domain and diff --git a/lib/who_need_help_web/controllers/metrics_controller.ex b/lib/who_need_help_web/controllers/metrics_controller.ex new file mode 100644 index 0000000..aef95b2 --- /dev/null +++ b/lib/who_need_help_web/controllers/metrics_controller.ex @@ -0,0 +1,31 @@ +defmodule WhoNeedHelpWeb.MetricsController do + use WhoNeedHelpWeb, :controller + + @content_type "text/plain; version=0.0.4" + + def show(conn, _params) do + token = Application.get_env(:who_need_help, :metrics_token) + + if authorized?(get_req_header(conn, "authorization"), token) do + body = TelemetryMetricsPrometheus.Core.scrape(:prometheus_metrics) + + conn + |> put_resp_header("cache-control", "no-store") + |> put_resp_content_type(@content_type) + |> send_resp(:ok, body) + else + conn + |> put_resp_header("cache-control", "no-store") + |> put_resp_header("www-authenticate", "Bearer") + |> send_resp(:unauthorized, "Unauthorized\n") + end + end + + defp authorized?(["Bearer " <> candidate], expected) + when is_binary(expected) and expected != "" and + byte_size(candidate) == byte_size(expected) do + Plug.Crypto.secure_compare(candidate, expected) + end + + defp authorized?(_authorization, _expected), do: false +end diff --git a/lib/who_need_help_web/router.ex b/lib/who_need_help_web/router.ex index bdc65e9..05044ff 100644 --- a/lib/who_need_help_web/router.ex +++ b/lib/who_need_help_web/router.ex @@ -33,6 +33,10 @@ defmodule WhoNeedHelpWeb.Router do get "/ready", HealthController, :ready end + scope "/", WhoNeedHelpWeb do + get "/metrics", MetricsController, :show + end + scope "/mobile", WhoNeedHelpWeb do pipe_through :mobile diff --git a/lib/who_need_help_web/telemetry.ex b/lib/who_need_help_web/telemetry.ex index 663d086..dad868e 100644 --- a/lib/who_need_help_web/telemetry.ex +++ b/lib/who_need_help_web/telemetry.ex @@ -8,17 +8,87 @@ defmodule WhoNeedHelpWeb.Telemetry do @impl true def init(_arg) do - children = [ - # Telemetry poller will execute the given period measurements - # every 10_000ms. Learn more here: https://telemetry-metrics.hexdocs.pm - {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} - # Add reporters as children of your supervision tree. - # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} - ] + reporter_children = + if Application.fetch_env!(:who_need_help, :app_role) == :web do + [ + {TelemetryMetricsPrometheus.Core, + name: :prometheus_metrics, metrics: prometheus_metrics(), start_async: false} + ] + else + [] + end + + children = + reporter_children ++ + [ + # Telemetry poller will execute the given period measurements + # every 10_000ms. Learn more here: https://telemetry-metrics.hexdocs.pm + {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} + ] Supervisor.init(children, strategy: :one_for_one) end + def prometheus_metrics do + [ + counter("who_need_help.http.requests.total", + event_name: [:phoenix, :endpoint, :stop], + measurement: :duration, + description: "Completed HTTP requests" + ), + sum("who_need_help.http.request.duration.microseconds.total", + event_name: [:phoenix, :endpoint, :stop], + measurement: fn measurements -> + System.convert_time_unit(measurements.duration, :native, :microsecond) + end, + description: "Cumulative HTTP request duration" + ), + counter("who_need_help.http.exceptions.total", + event_name: [:phoenix, :router_dispatch, :exception], + measurement: :duration, + description: "Router dispatch exceptions" + ), + counter("who_need_help.database.queries.total", + event_name: [:who_need_help, :repo, :query], + measurement: :total_time, + description: "Completed database queries" + ), + sum("who_need_help.database.query.duration.microseconds.total", + event_name: [:who_need_help, :repo, :query], + measurement: fn measurements -> + System.convert_time_unit(measurements.total_time, :native, :microsecond) + end, + description: "Cumulative database query duration" + ), + counter("who_need_help.websocket.connections.total", + event_name: [:phoenix, :socket_connected], + measurement: :duration, + description: "Completed WebSocket connection attempts" + ), + last_value("who_need_help.vm.memory.total.bytes", + event_name: [:vm, :memory], + measurement: :total, + unit: :byte, + description: "Total memory used by the Erlang VM" + ), + last_value("who_need_help.vm.run_queue.total", + event_name: [:vm, :total_run_queue_lengths], + measurement: :total, + description: "Total scheduler run queue length" + ), + last_value("who_need_help.vm.run_queue.cpu", + event_name: [:vm, :total_run_queue_lengths], + measurement: :cpu, + description: "CPU scheduler run queue length" + ), + last_value("who_need_help.vm.run_queue.io", + event_name: [:vm, :total_run_queue_lengths], + measurement: :io, + description: "I/O scheduler run queue length" + ) + ] + end + def metrics do [ # Phoenix Metrics diff --git a/mix.exs b/mix.exs index 1bdd8fc..faa9ae0 100644 --- a/mix.exs +++ b/mix.exs @@ -73,6 +73,7 @@ defmodule WhoNeedHelp.MixProject do {:req, "~> 0.5"}, {:assent, "~> 0.3.1"}, {:telemetry_metrics, "~> 1.0"}, + {:telemetry_metrics_prometheus_core, "~> 1.2.1"}, {:telemetry_poller, "~> 1.0"}, {:gettext, "~> 1.0"}, {:jason, "~> 1.2"}, diff --git a/mix.lock b/mix.lock index 759a3be..2d99885 100644 --- a/mix.lock +++ b/mix.lock @@ -47,6 +47,7 @@ "tailwind": {:hex, :tailwind, "0.5.1", "35435b13158c90d37da11e1cfc808755fca1d7b6c5ab87b1b19c5de87e2f0a10", [:mix], [], "hexpm", "c4e26302a59fec72abc5610ecb6ad2116d9aa31f31aab2d4b8eb6e95d25a689c"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"}, "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, diff --git a/scripts/backup-compose.sh b/scripts/backup-compose.sh new file mode 100755 index 0000000..df82460 --- /dev/null +++ b/scripts/backup-compose.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu +umask 077 + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +if [ ! -f "$ROOT/.env" ]; then + echo "Missing $ROOT/.env." >&2 + exit 1 +fi + +set -a +. "$ROOT/.env" +set +a + +: "${POSTGRES_DB:?Set POSTGRES_DB in .env}" +: "${POSTGRES_USER:?Set POSTGRES_USER in .env}" + +timestamp=$(date -u +%Y%m%d-%H%M%S) +target=${1:-"$ROOT/output/backups/compose-$timestamp.dump"} +target_dir=$(dirname -- "$target") +target_name=$(basename -- "$target") +partial="$target.partial.$$" +checksum="$target.sha256" +checksum_partial="$checksum.partial.$$" + +if [ -e "$target" ] || [ -e "$checksum" ]; then + echo "Refusing to overwrite $target or $checksum." >&2 + exit 1 +fi + +mkdir -p "$target_dir" + +cleanup() { + rm -f "$partial" "$checksum_partial" +} +trap cleanup EXIT HUP INT TERM + +docker compose exec -T db \ + pg_dump \ + --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" \ + --format custom \ + --no-owner >"$partial" + +test -s "$partial" + +docker compose exec -T db pg_restore --list <"$partial" >/dev/null + +hash=$(sha256sum "$partial" | awk '{print $1}') +printf '%s %s\n' "$hash" "$target_name" >"$checksum_partial" + +mv "$partial" "$target" +mv "$checksum_partial" "$checksum" +trap - EXIT HUP INT TERM + +echo "Backup: $target" +echo "Checksum: $checksum" +echo "SHA-256: $hash" diff --git a/scripts/ensure-local-metrics-token.sh b/scripts/ensure-local-metrics-token.sh new file mode 100755 index 0000000..6371d1b --- /dev/null +++ b/scripts/ensure-local-metrics-token.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +ENV_FILE="$ROOT/.env" + +if [ ! -f "$ENV_FILE" ]; then + echo "Missing $ENV_FILE. Copy .env.example to .env first." >&2 + exit 1 +fi + +for command in openssl perl; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "Required command is unavailable: $command" >&2 + exit 1 + fi +done + +current=$( + perl -ne ' + if (/^METRICS_TOKEN=(.*)$/) { + print $1; + exit; + } + ' "$ENV_FILE" +) + +if [ -n "$current" ]; then + chmod 600 "$ENV_FILE" + echo "METRICS_TOKEN already exists; no secret was changed." + exit 0 +fi + +new_metrics_token=$(openssl rand -hex 32) +tmp_env=$(mktemp "${ENV_FILE}.metrics.XXXXXX") +trap 'rm -f "$tmp_env"' EXIT HUP INT TERM +chmod 600 "$tmp_env" + +NEW_METRICS_TOKEN=$new_metrics_token perl -0pe ' + if (s/^METRICS_TOKEN=.*$/METRICS_TOKEN=$ENV{NEW_METRICS_TOKEN}/m) { + $replaced = 1; + } + + END { + print "\nMETRICS_TOKEN=$ENV{NEW_METRICS_TOKEN}\n" unless $replaced; + } +' "$ENV_FILE" >"$tmp_env" + +mv "$tmp_env" "$ENV_FILE" +chmod 600 "$ENV_FILE" +trap - EXIT HUP INT TERM +unset new_metrics_token + +echo "Generated METRICS_TOKEN in the ignored .env without printing it." diff --git a/scripts/kind-up.sh b/scripts/kind-up.sh index f71bf70..7fd3bd2 100755 --- a/scripts/kind-up.sh +++ b/scripts/kind-up.sh @@ -108,6 +108,7 @@ if ! kube --namespace "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; th secret_key_base=$(openssl rand -hex 64) handover_secret=$(openssl rand -hex 64) release_cookie=$(openssl rand -hex 64) + metrics_token=$(openssl rand -hex 32) kube --namespace "$NAMESPACE" create secret generic "$SECRET_NAME" \ --from-literal=POSTGRES_DB=who_need_help \ @@ -116,9 +117,18 @@ if ! kube --namespace "$NAMESPACE" get secret "$SECRET_NAME" >/dev/null 2>&1; th --from-literal="DATABASE_URL=ecto://postgres:${postgres_password}@postgis/who_need_help" \ --from-literal="SECRET_KEY_BASE=$secret_key_base" \ --from-literal="HANDOVER_SECRET=$handover_secret" \ - --from-literal="RELEASE_COOKIE=$release_cookie" + --from-literal="RELEASE_COOKIE=$release_cookie" \ + --from-literal="METRICS_TOKEN=$metrics_token" - unset postgres_password secret_key_base handover_secret release_cookie + unset postgres_password secret_key_base handover_secret release_cookie metrics_token +elif [ -z "$(kube --namespace "$NAMESPACE" get secret "$SECRET_NAME" -o jsonpath='{.data.METRICS_TOKEN}')" ]; then + metrics_token=$(openssl rand -hex 32) + + kube --namespace "$NAMESPACE" patch secret "$SECRET_NAME" \ + --type merge \ + --patch "{\"stringData\":{\"METRICS_TOKEN\":\"$metrics_token\"}}" >/dev/null + + unset metrics_token fi if [ -n "$legacy_backup" ]; then diff --git a/scripts/restore-drill-compose.sh b/scripts/restore-drill-compose.sh new file mode 100755 index 0000000..55d1157 --- /dev/null +++ b/scripts/restore-drill-compose.sh @@ -0,0 +1,169 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 PATH_TO_CUSTOM_FORMAT_DUMP" >&2 + exit 1 +fi + +dump=$1 +checksum="$dump.sha256" + +if [ ! -f "$dump" ] || [ ! -s "$dump" ]; then + echo "Dump does not exist or is empty: $dump" >&2 + exit 1 +fi + +if [ ! -f "$checksum" ]; then + echo "Checksum manifest is required: $checksum" >&2 + exit 1 +fi + +if [ ! -f "$ROOT/.env" ]; then + echo "Missing $ROOT/.env." >&2 + exit 1 +fi + +set -a +. "$ROOT/.env" +set +a + +: "${POSTGRES_DB:?Set POSTGRES_DB in .env}" +: "${POSTGRES_USER:?Set POSTGRES_USER in .env}" +: "${DATABASE_URL:?Set DATABASE_URL in .env}" + +dump_dir=$(CDPATH= cd -- "$(dirname -- "$dump")" && pwd) +dump_name=$(basename -- "$dump") + +( + cd "$dump_dir" + sha256sum --check --status "$(basename -- "$checksum")" +) + +docker compose exec -T db pg_restore --list <"$dump" >/dev/null + +drill_db="who_need_help_restore_drill_$(date -u +%Y%m%d%H%M%S)_$$" +created=false + +cleanup() { + if [ "$created" = true ]; then + docker compose exec -T db \ + dropdb --username "$POSTGRES_USER" --if-exists "$drill_db" >/dev/null + fi +} +trap cleanup EXIT HUP INT TERM + +docker compose exec -T db \ + createdb \ + --username "$POSTGRES_USER" \ + --template template0 \ + "$drill_db" +created=true + +docker compose exec -T db \ + pg_restore \ + --username "$POSTGRES_USER" \ + --dbname "$drill_db" \ + --exit-on-error \ + --no-owner \ + --no-privileges <"$dump" + +tables=$( + docker compose exec -T db \ + psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \ + --command " + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename <> 'spatial_ref_sys' + ORDER BY tablename; + " +) + +table_count=0 +total_rows=0 + +for table in $tables; do + restored_count=$( + docker compose exec -T db \ + psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \ + --command "SELECT count(*) FROM \"$table\";" + ) + + table_count=$((table_count + 1)) + total_rows=$((total_rows + restored_count)) +done + +if [ "$table_count" -eq 0 ]; then + echo "The restored database contains no public application tables." >&2 + exit 1 +fi + +postgis_version=$( + docker compose exec -T db \ + psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \ + --command "SELECT PostGIS_Lib_Version();" +) + +case "$DATABASE_URL" in + *\?*) + database_base=${DATABASE_URL%%\?*} + database_query="?${DATABASE_URL#*\?}" + ;; + + *) + database_base=$DATABASE_URL + database_query= + ;; +esac + +database_prefix=${database_base%/*} + +if [ "$database_prefix" = "$database_base" ]; then + echo "Could not derive the temporary drill DATABASE_URL." >&2 + exit 1 +fi + +drill_database_url="$database_prefix/$drill_db$database_query" + +docker compose run --rm --no-deps \ + --env APP_ROLE=migrate \ + --env "DATABASE_URL=$drill_database_url" \ + migrate /app/bin/migrate + +docker compose run --rm --no-deps \ + --env APP_ROLE=migrate \ + --env "DATABASE_URL=$drill_database_url" \ + migrate /app/bin/who_need_help eval 'WhoNeedHelp.Release.await_migrations()' + +migration_count=$( + docker compose exec -T db \ + psql --username "$POSTGRES_USER" --dbname "$drill_db" --tuples-only --no-align \ + --command "SELECT count(*) FROM schema_migrations;" +) + +docker compose exec -T db \ + dropdb --username "$POSTGRES_USER" "$drill_db" +created=false +trap - EXIT HUP INT TERM + +remaining=$( + docker compose exec -T db \ + psql --username "$POSTGRES_USER" --dbname postgres --tuples-only --no-align \ + --command "SELECT count(*) FROM pg_database WHERE datname = '$drill_db';" +) + +if [ "$remaining" != "0" ]; then + echo "Restore drill database was not removed: $drill_db" >&2 + exit 1 +fi + +echo "Restore drill passed for: $dump_name" +echo "Readable public tables: $table_count" +echo "Restored rows read: $total_rows" +echo "Applied migrations: $migration_count" +echo "PostGIS library: $postgis_version" +echo "Temporary database removed: $drill_db" diff --git a/test/who_need_help_web/controllers/metrics_controller_test.exs b/test/who_need_help_web/controllers/metrics_controller_test.exs new file mode 100644 index 0000000..f52ebf0 --- /dev/null +++ b/test/who_need_help_web/controllers/metrics_controller_test.exs @@ -0,0 +1,40 @@ +defmodule WhoNeedHelpWeb.MetricsControllerTest do + use WhoNeedHelpWeb.ConnCase, async: false + + test "rejects a request without credentials", %{conn: conn} do + conn = get(conn, "/metrics") + + assert response(conn, 401) == "Unauthorized\n" + assert get_resp_header(conn, "www-authenticate") == ["Bearer"] + assert get_resp_header(conn, "cache-control") == ["no-store"] + end + + test "rejects an incorrect bearer token", %{conn: conn} do + conn = + conn + |> put_req_header("authorization", "Bearer incorrect-token") + |> get("/metrics") + + assert response(conn, 401) == "Unauthorized\n" + end + + test "exports Prometheus metrics for the configured bearer token", %{conn: conn} do + get(conn, "/healthz/live") + + conn = + conn + |> put_req_header("authorization", "Bearer test-metrics-token") + |> get("/metrics") + + body = response(conn, 200) + + assert get_resp_header(conn, "content-type") == [ + "text/plain; version=0.0.4; charset=utf-8" + ] + + assert get_resp_header(conn, "cache-control") == ["no-store"] + assert body =~ "# TYPE who_need_help_http_requests_total counter" + assert body =~ "who_need_help_http_requests_total " + assert body =~ "who_need_help_http_request_duration_microseconds_total " + end +end