feat: add verified local observability stack

2026-07-19 07:11:29 +03:00 · 2026-07-19 07:11:29 +03:00 · 76a261b9ec
commit 76a261b9ec
parent 594065deb8
19 changed files with 1373 additions and 16 deletions
--- a/.env.load.example
+++ b/.env.load.example
@ -59,3 +59,11 @@ LOAD_FIXTURE_PASSWORD=GENERATE_LOAD_FIXTURE_PASSWORD
 LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=120
 LOAD_RESILIENCE_PROBE_INTERVAL_SECONDS=0.05
 LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=2
+OBSERVABILITY_PROMETHEUS_PORT=0
+OBSERVABILITY_ALERTMANAGER_PORT=0
+OBSERVABILITY_GRAFANA_PORT=0
+OBSERVABILITY_SCRAPE_INTERVAL=1s
+OBSERVABILITY_EVALUATION_INTERVAL=1s
+OBSERVABILITY_TIMEOUT_SECONDS=90
+OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin
+OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD
--- a/README.md
+++ b/README.md
@ -287,6 +287,27 @@ Compose crash/replacement and Oban retry checks use the separate load project:
 Both scripts retain ignored evidence under `output/resilience/`; their exact
 mutation and cleanup boundaries are documented in the operations runbook.

+Start and verify the local monitoring profile against every isolated load web
+replica:
+
+```bash
+./scripts/observability-run.sh local-observability
+```
+
+The command validates Prometheus and Alertmanager configuration, provisions the
+Grafana datasource and dashboard, discovers each current web container as a
+separate target, and exercises a firing/resolved alert by stopping and
+recovering exactly one verified load replica. It prints the loopback-only
+random ports and retains non-secret evidence below `output/observability/`.
+The generated Grafana password remains only in mode-`0600` `.env.load`.
+
+Stop only the monitoring services while leaving their local metric volumes and
+the load application running:
+
+```bash
+./scripts/observability-stop.sh
+```
+
 For an external cluster, provide a real PostgreSQL/PostGIS service and a
 pre-created Secret through required `existingSecret`; the chart never renders
 credentials from tracked values. The Secret must contain `DATABASE_URL`,
--- a/compose.observability.yaml
+++ b/compose.observability.yaml
@ -0,0 +1,136 @@
+services:
+  alert-receiver:
+    image: python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4
+    command: ["python", "/opt/who-need-help/alert-receiver.py"]
+    volumes:
+      - ./scripts/alert-receiver.py:/opt/who-need-help/alert-receiver.py:ro
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "python",
+          "-c",
+          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2).read()",
+        ]
+      interval: 2s
+      timeout: 3s
+      retries: 20
+    user: "65532:65532"
+    read_only: true
+    tmpfs:
+      - /tmp
+    cap_drop: [ALL]
+    security_opt:
+      - no-new-privileges:true
+    networks: [internal]
+    restart: unless-stopped
+    profiles: [observability]
+
+  alertmanager:
+    image: quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --storage.path=/alertmanager
+    volumes:
+      - ./ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager_data:/alertmanager
+    ports:
+      - target: 9093
+        published: "${OBSERVABILITY_ALERTMANAGER_PORT:-0}"
+        host_ip: 127.0.0.1
+        protocol: tcp
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:9093/-/ready"]
+      interval: 2s
+      timeout: 3s
+      retries: 20
+    depends_on:
+      alert-receiver:
+        condition: service_healthy
+    read_only: true
+    cap_drop: [ALL]
+    security_opt:
+      - no-new-privileges:true
+    networks: [internal, observability_host]
+    restart: unless-stopped
+    profiles: [observability]
+
+  prometheus:
+    image: quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893
+    command:
+      - --config.file=/runtime/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+    volumes:
+      - ${OBSERVABILITY_RUNTIME_DIR:?Set OBSERVABILITY_RUNTIME_DIR for the observability profile}/prometheus:/runtime:ro
+      - ./ops/observability/rules.yml:/etc/prometheus/rules.yml:ro
+      - prometheus_data:/prometheus
+    ports:
+      - target: 9090
+        published: "${OBSERVABILITY_PROMETHEUS_PORT:-0}"
+        host_ip: 127.0.0.1
+        protocol: tcp
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:9090/-/ready"]
+      interval: 2s
+      timeout: 3s
+      retries: 20
+    depends_on:
+      alertmanager:
+        condition: service_healthy
+    read_only: true
+    cap_drop: [ALL]
+    security_opt:
+      - no-new-privileges:true
+    networks: [internal, observability_host]
+    restart: unless-stopped
+    profiles: [observability]
+
+  grafana:
+    image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63
+    environment:
+      GF_SECURITY_ADMIN_USER: ${OBSERVABILITY_GRAFANA_ADMIN_USER:?Set OBSERVABILITY_GRAFANA_ADMIN_USER}
+      GF_SECURITY_ADMIN_PASSWORD__FILE: /run/wnh-secrets/admin-password
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+      GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
+      GF_PLUGINS_PREINSTALL_DISABLED: "true"
+      GF_UNIFIED_ALERTING_ENABLED: "false"
+    volumes:
+      - ${OBSERVABILITY_RUNTIME_DIR:?Set OBSERVABILITY_RUNTIME_DIR for the observability profile}/grafana:/run/wnh-secrets:ro
+      - ./ops/observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ./ops/observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
+      - ./ops/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana_data:/var/lib/grafana
+    ports:
+      - target: 3000
+        published: "${OBSERVABILITY_GRAFANA_PORT:-0}"
+        host_ip: 127.0.0.1
+        protocol: tcp
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3000/api/health"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+    depends_on:
+      prometheus:
+        condition: service_healthy
+    read_only: true
+    tmpfs:
+      - /tmp
+      - /var/log/grafana
+    cap_drop: [ALL]
+    security_opt:
+      - no-new-privileges:true
+    networks: [internal, observability_host]
+    restart: unless-stopped
+    profiles: [observability]
+
+networks:
+  observability_host:
+
+volumes:
+  alertmanager_data:
+  prometheus_data:
+  grafana_data:
--- a/docs/dependency-baseline.md
+++ b/docs/dependency-baseline.md
@ -1,6 +1,6 @@
 # Dependency baseline

-Verified on 2026-07-18. This is a point-in-time stable baseline, not a claim
+Verified through 2026-07-19. This is a point-in-time stable baseline, not a claim
 that future security updates or major-version migrations can be avoided.
 Application locks and OCI digests remain committed so the same revision can be
 rebuilt deterministically.
@ -38,6 +38,10 @@ package checksums are in `mix.lock` and `assets/package-lock.json`.
 | Traefik | 3.7.8 |
 | Mailpit | 1.30.4 |
 | k6 load generator | 2.1.0 |
+| Prometheus | 3.13.1 |
+| Alertmanager | 0.33.1 |
+| Grafana | 13.1.0 |
+| Python alert-boundary runtime | 3.14.6 / Alpine 3.23 |
 | Debian builder/runner snapshot | trixie-20260713-slim |

 Every external Compose/kind service image and every Dockerfile base image is
@ -81,9 +85,10 @@ because the official SDK channel identifies it as a QPR beta.
 | Sobelow | 0.14.1 |
 | GitHub checkout action | 7.0.0 / commit `9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0` |

-The four containerized command-line tools are pinned by both exact tag and OCI
-digest in `scripts/quality.sh`. The checkout action is pinned by commit in the
-workflow. Credo, Dialyxir, and Sobelow are locked in `mix.lock`.
+The four lint/security command-line images and the three observability
+validation images are pinned by both exact tag and OCI digest in
+`scripts/quality.sh`. The checkout action is pinned by commit in the workflow.
+Credo, Dialyxir, and Sobelow are locked in `mix.lock`.

 ## Primary version sources

@ -97,6 +102,10 @@ workflow. Credo, Dialyxir, and Sobelow are locked in `mix.lock`.
 - [Traefik releases](https://github.com/traefik/traefik/releases)
 - [Mailpit releases](https://github.com/axllent/mailpit/releases)
 - [k6 releases](https://github.com/grafana/k6/releases)
+- [Prometheus releases](https://github.com/prometheus/prometheus/releases)
+- [Alertmanager releases](https://github.com/prometheus/alertmanager/releases)
+- [Grafana releases](https://github.com/grafana/grafana/releases)
+- [Python releases](https://www.python.org/downloads/)
 - [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes)
 - [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk)
 - [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests)
@ -126,4 +135,5 @@ docker run --rm who-need-help:node-deps npm outdated --json
 ./scripts/e2e-run.sh
 ./scripts/android-build.sh
 ./scripts/android-instrumentation-test.sh
+./scripts/observability-run.sh local-observability
 ```
--- a/docs/local-hardening-plan.md
+++ b/docs/local-hardening-plan.md
@ -13,7 +13,7 @@ item below unless the evidence column explicitly describes a local mock.
 | Localization and accessibility | Completed locally: product copy and custom validation messages are extracted; EN/UK/RU catalogs and localized category descriptions/structured values are implemented | 508 default and 40 error messages are current; RU/UK have no empty/fuzzy entries; 163 backend tests and all 8 browser specs pass, including locale persistence, keyboard, axe, themes, responsive widths, and reconnect |
 | Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset |
 | Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data |
-| Observability | Protected Prometheus text endpoint exists | Local Prometheus/Grafana/Alertmanager profile scrapes every replica and an induced isolated failure exercises alert delivery |
+| Observability | Completed locally: protected per-process metrics feed a pinned Prometheus/Grafana/Alertmanager profile | All 3 direct web targets are up before/after the drill; a verified replica stop delivers firing and resolved webhooks; Grafana datasource/dashboard and an empty DB-count diff are retained |
 | Backup | Validated local custom-format dump and restore drill exist | An encrypted artifact is uploaded to local S3-compatible MinIO and restored into a fresh database; corruption and interrupted-upload checks fail closed |
 | External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout |
 | Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits |
@ -89,5 +89,10 @@ The goal remains open while any row lacks reproducible local evidence.
  The local single-node NodePort needed three reconnect attempts across 305
  ultimately successful samples; this is recorded rather than presented as
  raw transport continuity.
+- The isolated observability profile scrapes all three web containers by direct
+  target with a file-based Bearer credential, provisions a four-panel Grafana
+  dashboard, and routes the `up == 0` rule through Alertmanager. The canonical
+  drill observed both firing and resolved webhooks for the exact stopped
+  replica, restored all three targets, and left database counts unchanged.
 - The remaining rows above are still pending; this document is not a
  completion claim for the entire hardening goal.
--- a/docs/operations.md
+++ b/docs/operations.md
@ -134,17 +134,53 @@ integer-based; divide by `1_000_000` in PromQL when seconds are required.
 Definitions intentionally have no request path, user, request, or event-name
 labels that could create unbounded cardinality.

-Prometheus itself, durable metrics retention, alert rules, notification
-destinations, and measured alert thresholds are deployment responsibilities and
-are not claimed by this repository. In Kubernetes, put the token in
-`existingSecret`; configure the external scraper to send it as a Bearer token.
-
 Metrics are local to each BEAM process. Discover and scrape every web pod or
 container as a distinct target and preserve Prometheus's `instance` label. A
 request through the load-balanced public route reaches only one replica and is
 therefore useful as an authorization/smoke check, not as a cluster-wide
 aggregate.

+The isolated load project includes a local observability profile:
+
+```bash
+./scripts/load-stack-up.sh
+./scripts/observability-run.sh local-observability
+```
+
+The run script refuses the staging project, validates every current web
+container's Compose labels, and writes a `file_sd` target for each observed
+internal IP. Prometheus reads the Bearer value from a mode-`0600` runtime file,
+not a tracked config or URL. Its direct request includes the internal
+`X-Forwarded-Proto: https` signal required by the application's production SSL
+rewrite while preserving the target's own `instance` label.
+
+Prometheus, Alertmanager, and Grafana are pinned by tag and digest. Their host
+ports default to Docker-assigned values bound only to `127.0.0.1`; the run
+prints the observed URLs. Grafana uses the random admin password generated in
+ignored `.env.load`, disables anonymous signup, update checks, suggested plugin
+installation, and its unused built-in alert engine. The Prometheus datasource
+and four-panel dashboard are provisioned from tracked files.
+
+The verification stops exactly one scoped load web container. The
+`WhoNeedHelpWebReplicaUnavailable` rule is based only on the factual
+`up == 0` result; it is a local failure drill, not an invented latency,
+capacity, or production SLO threshold. The script requires both firing and
+resolved webhook payloads from Alertmanager, starts the same container, waits
+for every direct target, and compares read-only database counts before and
+after. Evidence is retained in `output/observability/` without the metrics or
+Grafana secrets.
+
+Stop only the monitoring services with:
+
+```bash
+./scripts/observability-stop.sh
+```
+
+Prometheus/Grafana/Alertmanager retention, production notification
+destinations, production availability, and measured alert policies remain
+deployment decisions. In Kubernetes, put the metrics token in
+`existingSecret`; configure the external scraper to send it as a Bearer token.
+
 ## Rollback boundary

 The release image is immutable and migrations run as a separate one-shot role.
--- a/docs/performance.md
+++ b/docs/performance.md
@ -204,6 +204,27 @@ Ignored evidence:
 - `output/resilience/compose-resilience-canonical-20260719f/`
 - `output/resilience/kind-rollout-canonical-20260719c/`

+## Observed local monitoring drill
+
+The canonical observability run on 2026-07-19 generated one direct Prometheus
+target for each of the three current load web containers. The exact expected
+and observed `instance` lists matched, and all three targets were `up` before
+the induced failure. Grafana reported a healthy Prometheus datasource and
+served the tracked provisioned dashboard with four panels.
+
+The script then stopped the exact scoped container
+`who_need_help_load-web-13`. Prometheus fired
+`WhoNeedHelpWebReplicaUnavailable`, and Alertmanager delivered a webhook whose
+labels identified that instance and the run. After the same container became
+healthy again, all three direct targets returned to `up`, the alert cleared,
+and a resolved webhook with the same instance/run labels arrived. The
+application-table count diff was empty. These are observed local protocol
+results, not a production monitoring or notification guarantee.
+
+Ignored evidence:
+
+- `output/observability/observability-canonical-20260719c/`
+
 Stop the isolated containers without deleting their database volume:

 ```sh
--- a/docs/verification.md
+++ b/docs/verification.md
@ -20,6 +20,7 @@ results from product limits and unknown production properties.
 | Voluntary thanks | Implemented as an external optional link | A helper can expose an optional link after completion; the UI states that the platform does not process the payment. | The platform does not provide payments, escrow, refunds, tax reporting, or payment guarantees. |
 | Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. |
 | Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. |
+| Local observability | Implemented and protocol-verified | Pinned Prometheus scraped all 3 direct load web targets with a file Bearer credential; Grafana provisioned a healthy datasource and four-panel dashboard; Alertmanager delivered firing and resolved webhooks for an induced scoped replica stop. | Local delivery does not establish production retention, notification-provider reliability, on-call policy, or measured alert thresholds. |

 ## Reproducible checks

@ -30,7 +31,7 @@ results from product limits and unknown production properties.
 - `mix compile --force --warnings-as-errors` and
  `mix format --check-formatted`: passed against the same final source.
 - `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning
-  threshold, actionlint 1.7.12, all four Compose renders, Helm lint, Trivy
+  threshold, actionlint 1.7.12, all five Compose renders, Helm lint, Trivy
  source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow
  strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the
  production-image vulnerability scan. The rendered Helm manifest and Debian
@ -78,6 +79,17 @@ results from product limits and unknown production properties.
  PubSub, and left the database-count diff empty. All 305 readiness samples
  ultimately returned 200; two samples needed three total reconnect attempts
  during local single-node NodePort endpoint replacement.
+- The canonical local observability drill matched all 3 generated/active
+  Prometheus instance targets, checked the provisioned Grafana datasource and
+  four-panel dashboard, received firing and resolved Alertmanager webhooks for
+  the exact stopped/recovered web replica, restored every target to `up`, and
+  left the application-table count diff empty. Its retained evidence contains
+  neither the metrics token nor the random Grafana password.
+- The observability stop command changed only the four scoped monitoring
+  container states. All app/database/worker container IDs stayed unchanged and
+  running, and the checked user/request/message counts were identical before
+  and after. A subsequent full drill returned all monitoring services to
+  healthy.
 - The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium
  specs against a fresh PostGIS volume with two web and two worker replicas on
  2026-07-19. The retained successful-run artifact directory is
--- a/ops/observability/alertmanager.yml
+++ b/ops/observability/alertmanager.yml
@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 1m
+
+route:
+  receiver: local-webhook
+  group_by: [alertname, instance, run_id]
+  group_wait: 0s
+  group_interval: 1s
+  repeat_interval: 1h
+
+receivers:
+  - name: local-webhook
+    webhook_configs:
+      - url: http://alert-receiver:8080/alerts
+        send_resolved: true
--- a/ops/observability/grafana/dashboards/who-need-help-overview.json
+++ b/ops/observability/grafana/dashboards/who-need-help-overview.json
@ -0,0 +1,193 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "wnh-prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum(up{job=\"who-need-help-web\"})",
+          "legendFormat": "available",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Available web replicas",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "wnh-prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 16,
+        "x": 8,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(who_need_help_http_requests_total[1m])",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP request rate by replica",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "wnh-prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "who_need_help_vm_memory_total_bytes",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "BEAM memory by replica",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "wnh-prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(who_need_help_database_queries_total[1m])",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Database query rate by replica",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 42,
+  "tags": ["who-need-help", "local"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Who Need Help local overview",
+  "uid": "wnh-overview",
+  "version": 1
+}
--- a/ops/observability/grafana/provisioning/dashboards/who-need-help.yml
+++ b/ops/observability/grafana/provisioning/dashboards/who-need-help.yml
@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: Who Need Help
+    orgId: 1
+    folder: Who Need Help
+    folderUid: wnh-local
+    type: file
+    disableDeletion: false
+    allowUiUpdates: false
+    updateIntervalSeconds: 10
+    options:
+      path: /var/lib/grafana/dashboards
--- a/ops/observability/grafana/provisioning/datasources/prometheus.yml
+++ b/ops/observability/grafana/provisioning/datasources/prometheus.yml
@ -0,0 +1,13 @@
+apiVersion: 1
+prune: true
+
+datasources:
+  - name: Who Need Help Prometheus
+    uid: wnh-prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      prometheusType: Prometheus
--- a/ops/observability/prometheus.template.yml
+++ b/ops/observability/prometheus.template.yml
@ -0,0 +1,27 @@
+global:
+  scrape_interval: __SCRAPE_INTERVAL__
+  evaluation_interval: __EVALUATION_INTERVAL__
+
+rule_files:
+  - /etc/prometheus/rules.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+scrape_configs:
+  - job_name: who-need-help-web
+    metrics_path: /metrics
+    authorization:
+      type: Bearer
+      credentials_file: /runtime/metrics-token
+    http_headers:
+      X-Forwarded-Proto:
+        values:
+          - https
+    file_sd_configs:
+      - files:
+          - /runtime/web-targets.json
+        refresh_interval: __SCRAPE_INTERVAL__
--- a/ops/observability/rules.yml
+++ b/ops/observability/rules.yml
@ -0,0 +1,10 @@
+groups:
+  - name: who-need-help-local-availability
+    rules:
+      - alert: WhoNeedHelpWebReplicaUnavailable
+        expr: up{job="who-need-help-web"} == 0
+        labels:
+          severity: local-drill
+        annotations:
+          summary: "The local Prometheus scraper cannot reach one web replica"
+          description: "The direct metrics target {{ $labels.instance }} is unavailable."
--- a/scripts/alert-receiver.py
+++ b/scripts/alert-receiver.py
@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Internal protocol-level webhook receiver for the local alert drill."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from http import HTTPStatus
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from threading import Lock
+from typing import ClassVar
+
+
+class AlertReceiver(BaseHTTPRequestHandler):
+    events: ClassVar[list[dict[str, object]]] = []
+    events_lock: ClassVar[Lock] = Lock()
+
+    def do_GET(self) -> None:
+        if self.path == "/healthz":
+            self._send_json(HTTPStatus.OK, {"status": "ok"})
+            return
+
+        if self.path == "/events":
+            with self.events_lock:
+                snapshot = list(self.events)
+
+            self._send_json(HTTPStatus.OK, snapshot)
+            return
+
+        self._send_json(HTTPStatus.NOT_FOUND, {"error": "not_found"})
+
+    def do_POST(self) -> None:
+        if self.path != "/alerts":
+            self._send_json(HTTPStatus.NOT_FOUND, {"error": "not_found"})
+            return
+
+        try:
+            content_length = int(self.headers.get("Content-Length", ""))
+            raw_payload = self.rfile.read(content_length)
+            payload = json.loads(raw_payload)
+        except (TypeError, ValueError, json.JSONDecodeError):
+            self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid_json"})
+            return
+
+        if not isinstance(payload, dict) or not isinstance(payload.get("alerts"), list):
+            self._send_json(HTTPStatus.UNPROCESSABLE_ENTITY, {"error": "invalid_alert_payload"})
+            return
+
+        event = {
+            "received_at": datetime.now(UTC).isoformat(),
+            "payload": payload,
+        }
+
+        with self.events_lock:
+            self.events.append(event)
+
+        self.send_response(HTTPStatus.NO_CONTENT)
+        self.end_headers()
+
+    def log_message(self, message: str, *args: object) -> None:
+        print(
+            f"{self.log_date_time_string()} {self.client_address[0]} "
+            f"{message % args}",
+            flush=True,
+        )
+
+    def _send_json(self, status: HTTPStatus, payload: object) -> None:
+        body = json.dumps(payload, separators=(",", ":")).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+
+if __name__ == "__main__":
+    ThreadingHTTPServer(("0.0.0.0", 8080), AlertReceiver).serve_forever()
--- a/scripts/ensure-local-load-env.sh
+++ b/scripts/ensure-local-load-env.sh
@ -20,6 +20,14 @@ if [ -f "$ENV_FILE" ]; then
  needs_resilience_interval=true
  needs_resilience_request_timeout=true
  needs_traefik_retry_attempts=true
+  needs_observability_prometheus_port=true
+  needs_observability_alertmanager_port=true
+  needs_observability_grafana_port=true
+  needs_observability_scrape_interval=true
+  needs_observability_evaluation_interval=true
+  needs_observability_timeout=true
+  needs_observability_grafana_user=true
+  needs_observability_grafana_password=true

  grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false
  grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" &&
@ -29,23 +37,52 @@ if [ -f "$ENV_FILE" ]; then
  grep -q '^LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=' "$ENV_FILE" &&
    needs_resilience_request_timeout=false
  grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$ENV_FILE" && needs_traefik_retry_attempts=false
+  grep -q '^OBSERVABILITY_PROMETHEUS_PORT=' "$ENV_FILE" &&
+    needs_observability_prometheus_port=false
+  grep -q '^OBSERVABILITY_ALERTMANAGER_PORT=' "$ENV_FILE" &&
+    needs_observability_alertmanager_port=false
+  grep -q '^OBSERVABILITY_GRAFANA_PORT=' "$ENV_FILE" &&
+    needs_observability_grafana_port=false
+  grep -q '^OBSERVABILITY_SCRAPE_INTERVAL=' "$ENV_FILE" &&
+    needs_observability_scrape_interval=false
+  grep -q '^OBSERVABILITY_EVALUATION_INTERVAL=' "$ENV_FILE" &&
+    needs_observability_evaluation_interval=false
+  grep -q '^OBSERVABILITY_TIMEOUT_SECONDS=' "$ENV_FILE" &&
+    needs_observability_timeout=false
+  grep -q '^OBSERVABILITY_GRAFANA_ADMIN_USER=' "$ENV_FILE" &&
+    needs_observability_grafana_user=false
+  grep -q '^OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=' "$ENV_FILE" &&
+    needs_observability_grafana_password=false

  if [ "$needs_fixture_password" = false ] &&
    [ "$needs_resilience_timeout" = false ] &&
    [ "$needs_resilience_interval" = false ] &&
    [ "$needs_resilience_request_timeout" = false ] &&
-    [ "$needs_traefik_retry_attempts" = false ]; then
+    [ "$needs_traefik_retry_attempts" = false ] &&
+    [ "$needs_observability_prometheus_port" = false ] &&
+    [ "$needs_observability_alertmanager_port" = false ] &&
+    [ "$needs_observability_grafana_port" = false ] &&
+    [ "$needs_observability_scrape_interval" = false ] &&
+    [ "$needs_observability_evaluation_interval" = false ] &&
+    [ "$needs_observability_timeout" = false ] &&
+    [ "$needs_observability_grafana_user" = false ] &&
+    [ "$needs_observability_grafana_password" = false ]; then
    echo ".env.load already exists; no secret or experiment input was changed."
    exit 0
  fi

  umask 077
  load_fixture_password=
+  observability_grafana_admin_password=

  if [ "$needs_fixture_password" = true ]; then
    load_fixture_password=$(openssl rand -hex 24)
  fi

+  if [ "$needs_observability_grafana_password" = true ]; then
+    observability_grafana_admin_password=$(openssl rand -hex 32)
+  fi
+
  {
    if [ "$needs_fixture_password" = true ]; then
      printf '\n# Added by the authenticated-load profile upgrade.\n'
@ -75,12 +112,61 @@ if [ -f "$ENV_FILE" ]; then
    if [ "$needs_traefik_retry_attempts" = true ]; then
      printf 'TRAEFIK_RETRY_ATTEMPTS=3\n'
    fi
+
+    if [ "$needs_observability_prometheus_port" = true ] ||
+      [ "$needs_observability_alertmanager_port" = true ] ||
+      [ "$needs_observability_grafana_port" = true ] ||
+      [ "$needs_observability_scrape_interval" = true ] ||
+      [ "$needs_observability_evaluation_interval" = true ] ||
+      [ "$needs_observability_timeout" = true ] ||
+      [ "$needs_observability_grafana_user" = true ] ||
+      [ "$needs_observability_grafana_password" = true ]; then
+      printf '\n# Added by the local observability-profile upgrade.\n'
+    fi
+
+    if [ "$needs_observability_prometheus_port" = true ]; then
+      printf 'OBSERVABILITY_PROMETHEUS_PORT=0\n'
+    fi
+
+    if [ "$needs_observability_alertmanager_port" = true ]; then
+      printf 'OBSERVABILITY_ALERTMANAGER_PORT=0\n'
+    fi
+
+    if [ "$needs_observability_grafana_port" = true ]; then
+      printf 'OBSERVABILITY_GRAFANA_PORT=0\n'
+    fi
+
+    if [ "$needs_observability_scrape_interval" = true ]; then
+      printf 'OBSERVABILITY_SCRAPE_INTERVAL=1s\n'
+    fi
+
+    if [ "$needs_observability_evaluation_interval" = true ]; then
+      printf 'OBSERVABILITY_EVALUATION_INTERVAL=1s\n'
+    fi
+
+    if [ "$needs_observability_timeout" = true ]; then
+      printf 'OBSERVABILITY_TIMEOUT_SECONDS=90\n'
+    fi
+
+    if [ "$needs_observability_grafana_user" = true ]; then
+      printf 'OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin\n'
+    fi
+
+    if [ "$needs_observability_grafana_password" = true ]; then
+      printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' \
+        "$observability_grafana_admin_password"
+    fi
  } >>"$ENV_FILE"
  chmod 600 "$ENV_FILE"
-  unset load_fixture_password needs_fixture_password needs_resilience_timeout \
+  unset load_fixture_password observability_grafana_admin_password \
+    needs_fixture_password needs_resilience_timeout \
    needs_resilience_interval needs_resilience_request_timeout \
-    needs_traefik_retry_attempts
-  echo "Added missing authenticated-load/resilience inputs to ignored .env.load."
+    needs_traefik_retry_attempts needs_observability_prometheus_port \
+    needs_observability_alertmanager_port needs_observability_grafana_port \
+    needs_observability_scrape_interval needs_observability_evaluation_interval \
+    needs_observability_timeout needs_observability_grafana_user \
+    needs_observability_grafana_password
+  echo "Added missing load/resilience/observability inputs to ignored .env.load."
  exit 0
 fi

@ -96,6 +182,7 @@ handover_secret=$(openssl rand -hex 64)
 release_cookie=$(openssl rand -hex 64)
 metrics_token=$(openssl rand -hex 32)
 load_fixture_password=$(openssl rand -hex 24)
+observability_grafana_admin_password=$(openssl rand -hex 32)
 database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load"
 temporary=$(mktemp "${ENV_FILE}.XXXXXX")
 trap 'rm -f "$temporary"' EXIT HUP INT TERM
@ -107,6 +194,7 @@ HANDOVER_SECRET_VALUE=$handover_secret \
 RELEASE_COOKIE_VALUE=$release_cookie \
 METRICS_TOKEN_VALUE=$metrics_token \
 LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \
+OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password \
  perl -0pe '
    s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g;
    s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g;
@ -115,6 +203,7 @@ LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \
    s/GENERATE_RELEASE_COOKIE/$ENV{RELEASE_COOKIE_VALUE}/g;
    s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g;
    s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g;
+    s/GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD/$ENV{OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE}/g;
  ' "$TEMPLATE" >"$temporary"

 if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then
@ -126,6 +215,6 @@ chmod 600 "$temporary"
 mv "$temporary" "$ENV_FILE"
 trap - EXIT HUP INT TERM
 unset postgres_password secret_key_base handover_secret release_cookie metrics_token \
-  load_fixture_password database_url
+  load_fixture_password observability_grafana_admin_password database_url

 echo "Generated independent load-profile secrets in ignored .env.load."
--- a/scripts/observability-run.sh
+++ b/scripts/observability-run.sh
@ -0,0 +1,573 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+ENV_FILE="$ROOT/.env.load"
+LABEL=${1:-"observability-$(date -u +%Y%m%dT%H%M%SZ)"}
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck source=/dev/null
+. "$ENV_FILE"
+set +a
+
+for name in LOAD_PROJECT LOAD_WEB_REPLICAS POSTGRES_DB METRICS_TOKEN \
+  OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \
+  OBSERVABILITY_GRAFANA_PORT OBSERVABILITY_SCRAPE_INTERVAL \
+  OBSERVABILITY_EVALUATION_INTERVAL OBSERVABILITY_TIMEOUT_SECONDS \
+  OBSERVABILITY_GRAFANA_ADMIN_USER OBSERVABILITY_GRAFANA_ADMIN_PASSWORD; do
+  if [[ -z "${!name:-}" ]]; then
+    echo "$name is missing from .env.load" >&2
+    exit 1
+  fi
+done
+
+if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
+  echo "The observability drill must not use the staging Compose project." >&2
+  exit 1
+fi
+
+if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
+  echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
+  exit 1
+fi
+
+if [[ ! "$LOAD_WEB_REPLICAS" =~ ^[1-9][0-9]*$ ]]; then
+  echo "LOAD_WEB_REPLICAS must be a positive integer." >&2
+  exit 1
+fi
+
+for name in OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \
+  OBSERVABILITY_GRAFANA_PORT; do
+  if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then
+    echo "$name must be a TCP port number from 0 through 65535." >&2
+    exit 1
+  fi
+done
+
+for name in OBSERVABILITY_SCRAPE_INTERVAL OBSERVABILITY_EVALUATION_INTERVAL; do
+  if [[ ! "${!name}" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then
+    echo "$name must be a positive Prometheus duration using ms, s, m, or h." >&2
+    exit 1
+  fi
+done
+
+if [[ ! "$OBSERVABILITY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
+  echo "OBSERVABILITY_TIMEOUT_SECONDS must be a positive integer." >&2
+  exit 1
+fi
+
+for command in curl docker jq sed; do
+  if ! command -v "$command" >/dev/null 2>&1; then
+    echo "Required command is unavailable: $command" >&2
+    exit 1
+  fi
+done
+
+runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT"
+output_dir="$ROOT/output/observability/$LABEL"
+prometheus_runtime="$runtime_dir/prometheus"
+grafana_runtime="$runtime_dir/grafana"
+mkdir -p "$prometheus_runtime" "$grafana_runtime" "$output_dir"
+chmod 700 "$ROOT/tmp" "$ROOT/tmp/observability" "$runtime_dir" \
+  "$ROOT/output" "$ROOT/output/observability" "$output_dir"
+
+export OBSERVABILITY_RUNTIME_DIR="$runtime_dir"
+
+set_runtime_owner() {
+  local directory=$1
+  local owner=$2
+
+  docker run --rm \
+    --user 0:0 \
+    --volume "$directory:/runtime" \
+    --entrypoint /bin/sh \
+    python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 \
+    -euc "chown -R $owner /runtime; chmod 700 /runtime"
+}
+
+host_owner="$(id -u):$(id -g)"
+set_runtime_owner "$prometheus_runtime" "$host_owner"
+set_runtime_owner "$grafana_runtime" "$host_owner"
+
+compose=(
+  docker compose
+  --env-file "$ENV_FILE"
+  -p "$LOAD_PROJECT"
+  -f compose.yaml
+  -f compose.load.yaml
+  -f compose.observability.yaml
+  --profile observability
+)
+
+service_ids() {
+  "${compose[@]}" ps --all -q "$1"
+}
+
+assert_scope() {
+  local container_id=$1
+  local expected_service=$2
+  local observed_project observed_service
+
+  observed_project=$(
+    docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
+      "$container_id"
+  )
+  observed_service=$(
+    docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
+      "$container_id"
+  )
+
+  if [[ "$observed_project" != "$LOAD_PROJECT" ||
+    "$observed_service" != "$expected_service" ]]; then
+    echo "Container scope mismatch for $container_id." >&2
+    exit 1
+  fi
+}
+
+wait_for_web() {
+  local container_id=$1
+  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    local state health
+    state=$(docker inspect --format '{{.State.Status}}' "$container_id")
+    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \
+      "$container_id")
+
+    if [[ "$state" == "running" && "$health" == "healthy" ]]; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for web container $container_id to become healthy." >&2
+  return 1
+}
+
+database_snapshot() {
+  # Variables are intentionally expanded inside the isolated PostGIS container.
+  # shellcheck disable=SC2016
+  "${compose[@]}" exec -T db sh -c \
+    'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
+      --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL'
+BEGIN READ ONLY;
+SELECT 'users' AS table_name, count(*) AS row_count FROM users
+UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
+UNION ALL SELECT 'help_requests', count(*) FROM help_requests
+UNION ALL SELECT 'messages', count(*) FROM messages
+UNION ALL SELECT 'categories', count(*) FROM categories
+UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
+UNION ALL SELECT 'activities', count(*) FROM activities
+UNION ALL SELECT 'reports', count(*) FROM reports
+UNION ALL SELECT 'social_identities', count(*) FROM social_identities
+UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
+UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
+UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations
+ORDER BY table_name;
+COMMIT;
+SQL
+}
+
+published_url() {
+  local service=$1
+  local target_port=$2
+  local published
+
+  published=$("${compose[@]}" port "$service" "$target_port" | head -n 1)
+
+  if [[ ! "$published" =~ ^127[.]0[.]0[.]1:([1-9][0-9]*)$ ]]; then
+    echo "The $service port is not bound to an observed loopback port." >&2
+    return 1
+  fi
+
+  printf 'http://%s' "$published"
+}
+
+wait_for_http() {
+  local url=$1
+  shift
+  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    if curl --fail --silent --show-error "$@" "$url" >/dev/null 2>&1; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for $url." >&2
+  return 1
+}
+
+fetch_receiver_events() {
+  local receiver_id
+  receiver_id=$(service_ids alert-receiver | head -n 1)
+  assert_scope "$receiver_id" alert-receiver
+  docker exec "$receiver_id" \
+    python -c \
+    'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/events", timeout=2).read().decode())'
+}
+
+wait_for_targets_up() {
+  local expected=$1
+  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    if curl --fail --silent --show-error \
+      "$prometheus_url/api/v1/targets?state=active" \
+      >"$output_dir/targets-current.json" 2>/dev/null &&
+      jq -e --argjson expected "$expected" --arg run_id "$LABEL" '
+        [
+          .data.activeTargets[]
+          | select(
+              .labels.job == "who-need-help-web" and
+              .labels.run_id == $run_id
+            )
+        ] as $targets
+        | ($targets | length) == $expected
+          and all($targets[]; .health == "up")
+      ' "$output_dir/targets-current.json" >/dev/null; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for every direct web metrics target." >&2
+  return 1
+}
+
+wait_for_firing_alert() {
+  local instance=$1
+  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \
+      >"$output_dir/prometheus-alerts-firing.json" 2>/dev/null &&
+      jq -e --arg instance "$instance" --arg run_id "$LABEL" '
+        any(
+          .data.alerts[];
+          .state == "firing" and
+          .labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and
+          .labels.instance == $instance and
+          .labels.run_id == $run_id
+        )
+      ' "$output_dir/prometheus-alerts-firing.json" >/dev/null; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for the induced Prometheus alert." >&2
+  return 1
+}
+
+wait_for_webhook_status() {
+  local instance=$1
+  local expected_status=$2
+  local destination=$3
+  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    if fetch_receiver_events >"$destination" 2>/dev/null &&
+      jq -e \
+        --arg instance "$instance" \
+        --arg run_id "$LABEL" \
+        --arg expected_status "$expected_status" '
+          any(
+            .[].payload.alerts[];
+            .status == $expected_status and
+            .labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and
+            .labels.instance == $instance and
+            .labels.run_id == $run_id
+          )
+        ' "$destination" >/dev/null; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for the $expected_status Alertmanager webhook." >&2
+  return 1
+}
+
+wait_for_alert_clear() {
+  local instance=$1
+  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
+
+  while ((SECONDS < deadline)); do
+    if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \
+      >"$output_dir/prometheus-alerts-resolved.json" 2>/dev/null &&
+      jq -e --arg instance "$instance" --arg run_id "$LABEL" '
+        all(
+          .data.alerts[];
+          .labels.alertname != "WhoNeedHelpWebReplicaUnavailable" or
+          .labels.instance != $instance or
+          .labels.run_id != $run_id or
+          .state != "firing"
+        )
+      ' "$output_dir/prometheus-alerts-resolved.json" >/dev/null; then
+      return 0
+    fi
+
+    sleep 1
+  done
+
+  echo "Timed out waiting for the induced Prometheus alert to resolve." >&2
+  return 1
+}
+
+restore_web() {
+  if [[ -n "${drill_web_id:-}" ]]; then
+    local state
+    state=$(docker inspect --format '{{.State.Status}}' "$drill_web_id" 2>/dev/null || true)
+
+    if [[ "$state" != "running" ]]; then
+      docker start "$drill_web_id" >/dev/null 2>&1 || true
+    fi
+
+    wait_for_web "$drill_web_id" >/dev/null 2>&1 || true
+  fi
+}
+
+cleanup() {
+  local status=$?
+  trap - EXIT HUP INT TERM
+  restore_web
+  exit "$status"
+}
+
+trap cleanup EXIT HUP INT TERM
+
+mapfile -t web_ids < <(service_ids web)
+
+if [[ "${#web_ids[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then
+  echo "Expected $LOAD_WEB_REPLICAS running load web replicas; observed ${#web_ids[@]}." >&2
+  exit 1
+fi
+
+internal_network=$(
+  docker network ls \
+    --filter "label=com.docker.compose.project=$LOAD_PROJECT" \
+    --filter "label=com.docker.compose.network=internal" \
+    --format '{{.Name}}'
+)
+
+if [[ -z "$internal_network" || "$internal_network" == *$'\n'* ]]; then
+  echo "Expected exactly one observed Compose internal network." >&2
+  exit 1
+fi
+
+target_lines="$prometheus_runtime/web-targets.jsonl"
+: >"$target_lines"
+
+for web_id in "${web_ids[@]}"; do
+  assert_scope "$web_id" web
+  wait_for_web "$web_id"
+  web_name=$(docker inspect --format '{{.Name}}' "$web_id")
+  web_name=${web_name#/}
+  web_ip=$(
+    docker inspect \
+      --format "{{with index .NetworkSettings.Networks \"$internal_network\"}}{{.IPAddress}}{{end}}" \
+      "$web_id"
+  )
+
+  if [[ ! "$web_ip" =~ ^[0-9]+([.][0-9]+){3}$ ]]; then
+    echo "No observed IPv4 address for $web_name on $internal_network." >&2
+    exit 1
+  fi
+
+  jq -cn \
+    --arg target "$web_ip:4000" \
+    --arg instance "$web_name" \
+    --arg compose_project "$LOAD_PROJECT" \
+    --arg run_id "$LABEL" \
+    '{
+      targets: [$target],
+      labels: {
+        instance: $instance,
+        compose_project: $compose_project,
+        run_id: $run_id
+      }
+    }' >>"$target_lines"
+done
+
+jq -s '.' "$target_lines" >"$prometheus_runtime/web-targets.json"
+unlink "$target_lines"
+
+sed \
+  -e "s/__SCRAPE_INTERVAL__/$OBSERVABILITY_SCRAPE_INTERVAL/g" \
+  -e "s/__EVALUATION_INTERVAL__/$OBSERVABILITY_EVALUATION_INTERVAL/g" \
+  "$ROOT/ops/observability/prometheus.template.yml" \
+  >"$prometheus_runtime/prometheus.yml"
+printf '%s' "$METRICS_TOKEN" >"$prometheus_runtime/metrics-token"
+printf '%s' "$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
+  >"$grafana_runtime/admin-password"
+cp "$prometheus_runtime/web-targets.json" "$output_dir/generated-targets.json"
+chmod 600 "$prometheus_runtime/prometheus.yml" \
+  "$prometheus_runtime/web-targets.json" \
+  "$prometheus_runtime/metrics-token" \
+  "$grafana_runtime/admin-password" \
+  "$output_dir/generated-targets.json"
+set_runtime_owner "$prometheus_runtime" "65534:65534"
+set_runtime_owner "$grafana_runtime" "472:0"
+
+docker run --rm \
+  --volume "$prometheus_runtime:/runtime:ro" \
+  --volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \
+  --entrypoint /bin/promtool \
+  quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 \
+  check config /runtime/prometheus.yml \
+  >"$output_dir/promtool-check.txt"
+
+docker run --rm \
+  --volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \
+  --entrypoint /bin/amtool \
+  quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d \
+  check-config /etc/alertmanager/alertmanager.yml \
+  >"$output_dir/amtool-check.txt"
+
+jq --exit-status 'type == "object" and .uid == "wnh-overview"' \
+  "$ROOT/ops/observability/grafana/dashboards/who-need-help-overview.json" \
+  >/dev/null
+
+for service in alert-receiver alertmanager prometheus grafana; do
+  while IFS= read -r existing_id; do
+    [[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service"
+  done < <(service_ids "$service")
+done
+
+database_snapshot "$output_dir/database-before.txt"
+
+"${compose[@]}" up -d --force-recreate --wait \
+  alert-receiver alertmanager prometheus grafana \
+  >"$output_dir/compose-up.txt"
+
+for service in alert-receiver alertmanager prometheus grafana; do
+  service_id=$(service_ids "$service" | head -n 1)
+  assert_scope "$service_id" "$service"
+done
+
+prometheus_url=$(published_url prometheus 9090)
+alertmanager_url=$(published_url alertmanager 9093)
+grafana_url=$(published_url grafana 3000)
+
+wait_for_http "$prometheus_url/-/ready"
+wait_for_http "$alertmanager_url/-/ready"
+wait_for_http "$grafana_url/api/health"
+wait_for_http "$grafana_url/api/datasources/uid/wnh-prometheus/health" \
+  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD"
+
+curl --fail --silent --show-error \
+  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
+  "$grafana_url/api/health" >"$output_dir/grafana-health.json"
+curl --fail --silent --show-error \
+  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
+  "$grafana_url/api/datasources/uid/wnh-prometheus/health" \
+  >"$output_dir/grafana-datasource-health.json"
+curl --fail --silent --show-error \
+  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
+  "$grafana_url/api/dashboards/uid/wnh-overview" \
+  >"$output_dir/grafana-dashboard.json"
+
+if ! jq -e '
+  .dashboard.uid == "wnh-overview" and
+  .meta.provisioned == true and
+  (.dashboard.panels | length) == 4
+' "$output_dir/grafana-dashboard.json" >/dev/null; then
+  echo "The provisioned Grafana dashboard did not match the tracked dashboard." >&2
+  exit 1
+fi
+
+wait_for_targets_up "$LOAD_WEB_REPLICAS"
+mv "$output_dir/targets-current.json" "$output_dir/targets-before-drill.json"
+
+jq -n \
+  --slurpfile expected "$output_dir/generated-targets.json" \
+  --slurpfile observed "$output_dir/targets-before-drill.json" '
+    {
+      expected_instances: ($expected[0] | map(.labels.instance) | sort),
+      observed_instances: (
+        $observed[0].data.activeTargets
+        | map(select(.labels.job == "who-need-help-web") | .labels.instance)
+        | sort
+      )
+    }
+    | . + {
+        exact_instance_match: (.expected_instances == .observed_instances)
+      }
+  ' >"$output_dir/target-summary.json"
+
+if ! jq -e '.exact_instance_match' "$output_dir/target-summary.json" >/dev/null; then
+  echo "Prometheus did not preserve every direct web instance target." >&2
+  exit 1
+fi
+
+drill_web_id=${web_ids[0]}
+assert_scope "$drill_web_id" web
+drill_instance=$(docker inspect --format '{{.Name}}' "$drill_web_id")
+drill_instance=${drill_instance#/}
+docker stop --time 30 "$drill_web_id" >"$output_dir/stopped-web.txt"
+
+wait_for_firing_alert "$drill_instance"
+wait_for_webhook_status \
+  "$drill_instance" firing "$output_dir/alert-webhooks-firing.json"
+
+docker start "$drill_web_id" >"$output_dir/started-web.txt"
+wait_for_web "$drill_web_id"
+wait_for_targets_up "$LOAD_WEB_REPLICAS"
+mv "$output_dir/targets-current.json" "$output_dir/targets-after-recovery.json"
+wait_for_alert_clear "$drill_instance"
+wait_for_webhook_status \
+  "$drill_instance" resolved "$output_dir/alert-webhooks-complete.json"
+
+curl --fail --silent --show-error "$alertmanager_url/api/v2/alerts" \
+  >"$output_dir/alertmanager-alerts-after-recovery.json"
+database_snapshot "$output_dir/database-after.txt"
+
+if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \
+  >"$output_dir/database-diff.txt"; then
+  echo "The observability drill changed tracked database counts." >&2
+  exit 1
+fi
+
+"${compose[@]}" ps -a >"$output_dir/compose-after.txt"
+"${compose[@]}" images --format json >"$output_dir/images.json"
+
+jq -n \
+  --arg run_id "$LABEL" \
+  --arg drill_instance "$drill_instance" \
+  --arg prometheus_url "$prometheus_url" \
+  --arg alertmanager_url "$alertmanager_url" \
+  --arg grafana_url "$grafana_url" \
+  --argjson web_replicas "$LOAD_WEB_REPLICAS" \
+  '{
+    run_id: $run_id,
+    web_replicas: $web_replicas,
+    direct_targets_up_before_and_after: true,
+    induced_instance: $drill_instance,
+    firing_webhook_observed: true,
+    resolved_webhook_observed: true,
+    database_count_diff_bytes: 0,
+    prometheus_url: $prometheus_url,
+    alertmanager_url: $alertmanager_url,
+    grafana_url: $grafana_url,
+    grafana_dashboard_path: "/d/wnh-overview/overview"
+  }' >"$output_dir/summary.json"
+
+trap - EXIT HUP INT TERM
+
+printf 'Observability evidence: %s\n' "$output_dir"
+printf 'Prometheus: %s\nAlertmanager: %s\nGrafana: %s/d/wnh-overview/overview\n' \
+  "$prometheus_url" "$alertmanager_url" "$grafana_url"
+printf 'Grafana user: %s; its random password remains only in ignored .env.load.\n' \
+  "$OBSERVABILITY_GRAFANA_ADMIN_USER"
--- a/scripts/observability-stop.sh
+++ b/scripts/observability-stop.sh
@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+ENV_FILE="$ROOT/.env.load"
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Missing $ENV_FILE; no observability project was selected." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck source=/dev/null
+. "$ENV_FILE"
+set +a
+
+: "${LOAD_PROJECT:?LOAD_PROJECT is missing from .env.load}"
+
+if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
+  echo "Refusing to stop services in the staging Compose project." >&2
+  exit 1
+fi
+
+runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT"
+export OBSERVABILITY_RUNTIME_DIR="$runtime_dir"
+
+compose=(
+  docker compose
+  --env-file "$ENV_FILE"
+  -p "$LOAD_PROJECT"
+  -f compose.yaml
+  -f compose.load.yaml
+  -f compose.observability.yaml
+  --profile observability
+)
+
+for service in alert-receiver alertmanager prometheus grafana; do
+  while IFS= read -r container_id; do
+    [[ -z "$container_id" ]] && continue
+    observed_project=$(
+      docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
+        "$container_id"
+    )
+    observed_service=$(
+      docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
+        "$container_id"
+    )
+
+    if [[ "$observed_project" != "$LOAD_PROJECT" ||
+      "$observed_service" != "$service" ]]; then
+      echo "Container scope mismatch for $container_id." >&2
+      exit 1
+    fi
+  done < <("${compose[@]}" ps --all -q "$service")
+done
+
+"${compose[@]}" stop grafana prometheus alertmanager alert-receiver
+
+echo "Stopped only the isolated observability services; app and metric volumes remain."
--- a/scripts/quality.sh
+++ b/scripts/quality.sh
@ -8,6 +8,9 @@ SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.11.0@sha256:9955be09ea7f0dbf7ae9
 HADOLINT_IMAGE="hadolint/hadolint:v2.14.0-debian@sha256:158cd0184dcaa18bd8ec20b61f4c1cabdf8b32a592d062f57bdcb8e4c1d312e2"
 ACTIONLINT_IMAGE="rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667"
 TRIVY_IMAGE="aquasec/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f"
+PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893"
+ALERTMANAGER_IMAGE="quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d"
+PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4"

 run_id="$(date -u +%Y%m%d%H%M%S)-$$"
 project="wnh_quality_$(printf '%s' "$run_id" | tr -d '-')"
@ -64,8 +67,44 @@ E2E_OUTPUT_DIR="$scan_dir/e2e-output" docker compose --env-file .env.e2e.example
  -f compose.yaml -f compose.e2e.yaml config --quiet
 docker compose --env-file .env.load.example \
  -f compose.yaml -f compose.load.yaml config --quiet
+mkdir -p "$scan_dir/observability-runtime/prometheus" \
+  "$scan_dir/observability-runtime/grafana"
+OBSERVABILITY_RUNTIME_DIR="$scan_dir/observability-runtime" \
+  docker compose --env-file .env.load.example \
+  -f compose.yaml -f compose.load.yaml -f compose.observability.yaml \
+  --profile observability config --quiet
 docker compose -p "$project" -f compose.quality.yaml config --quiet

+echo "Validating local observability configuration"
+sed \
+  -e 's/__SCRAPE_INTERVAL__/1s/g' \
+  -e 's/__EVALUATION_INTERVAL__/1s/g' \
+  ops/observability/prometheus.template.yml \
+  >"$scan_dir/observability-runtime/prometheus/prometheus.yml"
+printf '%s' 'isolated-quality-metrics-token' \
+  >"$scan_dir/observability-runtime/prometheus/metrics-token"
+printf '%s\n' '[]' \
+  >"$scan_dir/observability-runtime/prometheus/web-targets.json"
+docker run --rm \
+  --user 0:0 \
+  --volume "$scan_dir/observability-runtime/prometheus:/runtime:ro" \
+  --volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \
+  --entrypoint /bin/promtool \
+  "$PROMETHEUS_IMAGE" check config /runtime/prometheus.yml
+docker run --rm \
+  --user 0:0 \
+  --volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \
+  --entrypoint /bin/amtool \
+  "$ALERTMANAGER_IMAGE" check-config /etc/alertmanager/alertmanager.yml
+docker run --rm \
+  --volume "$ROOT/scripts/alert-receiver.py:/src/alert-receiver.py:ro" \
+  "$PYTHON_IMAGE" python -c \
+  'import py_compile; py_compile.compile("/src/alert-receiver.py", cfile="/tmp/alert-receiver.pyc", doraise=True)'
+jq --exit-status \
+  'type == "object" and .uid == "wnh-overview" and (.panels | length) == 4' \
+  ops/observability/grafana/dashboards/who-need-help-overview.json \
+  >/dev/null
+
 echo "Linting the Helm chart"
 "$ROOT/scripts/bootstrap-kubernetes-tools.sh" >/dev/null
 "$ROOT/.tools/bin/helm" lint \