diff --git a/.env.load.example b/.env.load.example index 0fe2356..385e82e 100644 --- a/.env.load.example +++ b/.env.load.example @@ -59,3 +59,11 @@ LOAD_FIXTURE_PASSWORD=GENERATE_LOAD_FIXTURE_PASSWORD LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=120 LOAD_RESILIENCE_PROBE_INTERVAL_SECONDS=0.05 LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=2 +OBSERVABILITY_PROMETHEUS_PORT=0 +OBSERVABILITY_ALERTMANAGER_PORT=0 +OBSERVABILITY_GRAFANA_PORT=0 +OBSERVABILITY_SCRAPE_INTERVAL=1s +OBSERVABILITY_EVALUATION_INTERVAL=1s +OBSERVABILITY_TIMEOUT_SECONDS=90 +OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin +OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD diff --git a/README.md b/README.md index 32972e1..3e1e993 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,27 @@ Compose crash/replacement and Oban retry checks use the separate load project: Both scripts retain ignored evidence under `output/resilience/`; their exact mutation and cleanup boundaries are documented in the operations runbook. +Start and verify the local monitoring profile against every isolated load web +replica: + +```bash +./scripts/observability-run.sh local-observability +``` + +The command validates Prometheus and Alertmanager configuration, provisions the +Grafana datasource and dashboard, discovers each current web container as a +separate target, and exercises a firing/resolved alert by stopping and +recovering exactly one verified load replica. It prints the loopback-only +random ports and retains non-secret evidence below `output/observability/`. +The generated Grafana password remains only in mode-`0600` `.env.load`. + +Stop only the monitoring services while leaving their local metric volumes and +the load application running: + +```bash +./scripts/observability-stop.sh +``` + For an external cluster, provide a real PostgreSQL/PostGIS service and a pre-created Secret through required `existingSecret`; the chart never renders credentials from tracked values. The Secret must contain `DATABASE_URL`, diff --git a/compose.observability.yaml b/compose.observability.yaml new file mode 100644 index 0000000..cef4c6c --- /dev/null +++ b/compose.observability.yaml @@ -0,0 +1,136 @@ +services: + alert-receiver: + image: python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 + command: ["python", "/opt/who-need-help/alert-receiver.py"] + volumes: + - ./scripts/alert-receiver.py:/opt/who-need-help/alert-receiver.py:ro + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2).read()", + ] + interval: 2s + timeout: 3s + retries: 20 + user: "65532:65532" + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal] + restart: unless-stopped + profiles: [observability] + + alertmanager: + image: quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d + command: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + volumes: + - ./ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager_data:/alertmanager + ports: + - target: 9093 + published: "${OBSERVABILITY_ALERTMANAGER_PORT:-0}" + host_ip: 127.0.0.1 + protocol: tcp + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:9093/-/ready"] + interval: 2s + timeout: 3s + retries: 20 + depends_on: + alert-receiver: + condition: service_healthy + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal, observability_host] + restart: unless-stopped + profiles: [observability] + + prometheus: + image: quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 + command: + - --config.file=/runtime/prometheus.yml + - --storage.tsdb.path=/prometheus + volumes: + - ${OBSERVABILITY_RUNTIME_DIR:?Set OBSERVABILITY_RUNTIME_DIR for the observability profile}/prometheus:/runtime:ro + - ./ops/observability/rules.yml:/etc/prometheus/rules.yml:ro + - prometheus_data:/prometheus + ports: + - target: 9090 + published: "${OBSERVABILITY_PROMETHEUS_PORT:-0}" + host_ip: 127.0.0.1 + protocol: tcp + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:9090/-/ready"] + interval: 2s + timeout: 3s + retries: 20 + depends_on: + alertmanager: + condition: service_healthy + read_only: true + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal, observability_host] + restart: unless-stopped + profiles: [observability] + + grafana: + image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63 + environment: + GF_SECURITY_ADMIN_USER: ${OBSERVABILITY_GRAFANA_ADMIN_USER:?Set OBSERVABILITY_GRAFANA_ADMIN_USER} + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/wnh-secrets/admin-password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + GF_UNIFIED_ALERTING_ENABLED: "false" + volumes: + - ${OBSERVABILITY_RUNTIME_DIR:?Set OBSERVABILITY_RUNTIME_DIR for the observability profile}/grafana:/run/wnh-secrets:ro + - ./ops/observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./ops/observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./ops/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + ports: + - target: 3000 + published: "${OBSERVABILITY_GRAFANA_PORT:-0}" + host_ip: 127.0.0.1 + protocol: tcp + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3000/api/health"] + interval: 2s + timeout: 3s + retries: 30 + depends_on: + prometheus: + condition: service_healthy + read_only: true + tmpfs: + - /tmp + - /var/log/grafana + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + networks: [internal, observability_host] + restart: unless-stopped + profiles: [observability] + +networks: + observability_host: + +volumes: + alertmanager_data: + prometheus_data: + grafana_data: diff --git a/docs/dependency-baseline.md b/docs/dependency-baseline.md index fa247d6..383318a 100644 --- a/docs/dependency-baseline.md +++ b/docs/dependency-baseline.md @@ -1,6 +1,6 @@ # Dependency baseline -Verified on 2026-07-18. This is a point-in-time stable baseline, not a claim +Verified through 2026-07-19. This is a point-in-time stable baseline, not a claim that future security updates or major-version migrations can be avoided. Application locks and OCI digests remain committed so the same revision can be rebuilt deterministically. @@ -38,6 +38,10 @@ package checksums are in `mix.lock` and `assets/package-lock.json`. | Traefik | 3.7.8 | | Mailpit | 1.30.4 | | k6 load generator | 2.1.0 | +| Prometheus | 3.13.1 | +| Alertmanager | 0.33.1 | +| Grafana | 13.1.0 | +| Python alert-boundary runtime | 3.14.6 / Alpine 3.23 | | Debian builder/runner snapshot | trixie-20260713-slim | Every external Compose/kind service image and every Dockerfile base image is @@ -81,9 +85,10 @@ because the official SDK channel identifies it as a QPR beta. | Sobelow | 0.14.1 | | GitHub checkout action | 7.0.0 / commit `9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0` | -The four containerized command-line tools are pinned by both exact tag and OCI -digest in `scripts/quality.sh`. The checkout action is pinned by commit in the -workflow. Credo, Dialyxir, and Sobelow are locked in `mix.lock`. +The four lint/security command-line images and the three observability +validation images are pinned by both exact tag and OCI digest in +`scripts/quality.sh`. The checkout action is pinned by commit in the workflow. +Credo, Dialyxir, and Sobelow are locked in `mix.lock`. ## Primary version sources @@ -97,6 +102,10 @@ workflow. Credo, Dialyxir, and Sobelow are locked in `mix.lock`. - [Traefik releases](https://github.com/traefik/traefik/releases) - [Mailpit releases](https://github.com/axllent/mailpit/releases) - [k6 releases](https://github.com/grafana/k6/releases) +- [Prometheus releases](https://github.com/prometheus/prometheus/releases) +- [Alertmanager releases](https://github.com/prometheus/alertmanager/releases) +- [Grafana releases](https://github.com/grafana/grafana/releases) +- [Python releases](https://www.python.org/downloads/) - [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes) - [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk) - [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests) @@ -126,4 +135,5 @@ docker run --rm who-need-help:node-deps npm outdated --json ./scripts/e2e-run.sh ./scripts/android-build.sh ./scripts/android-instrumentation-test.sh +./scripts/observability-run.sh local-observability ``` diff --git a/docs/local-hardening-plan.md b/docs/local-hardening-plan.md index 4d64739..7991c34 100644 --- a/docs/local-hardening-plan.md +++ b/docs/local-hardening-plan.md @@ -13,7 +13,7 @@ item below unless the evidence column explicitly describes a local mock. | Localization and accessibility | Completed locally: product copy and custom validation messages are extracted; EN/UK/RU catalogs and localized category descriptions/structured values are implemented | 508 default and 40 error messages are current; RU/UK have no empty/fuzzy entries; 163 backend tests and all 8 browser specs pass, including locale persistence, keyboard, axe, themes, responsive widths, and reconnect | | Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset | | Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data | -| Observability | Protected Prometheus text endpoint exists | Local Prometheus/Grafana/Alertmanager profile scrapes every replica and an induced isolated failure exercises alert delivery | +| Observability | Completed locally: protected per-process metrics feed a pinned Prometheus/Grafana/Alertmanager profile | All 3 direct web targets are up before/after the drill; a verified replica stop delivers firing and resolved webhooks; Grafana datasource/dashboard and an empty DB-count diff are retained | | Backup | Validated local custom-format dump and restore drill exist | An encrypted artifact is uploaded to local S3-compatible MinIO and restored into a fresh database; corruption and interrupted-upload checks fail closed | | External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout | | Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits | @@ -89,5 +89,10 @@ The goal remains open while any row lacks reproducible local evidence. The local single-node NodePort needed three reconnect attempts across 305 ultimately successful samples; this is recorded rather than presented as raw transport continuity. +- The isolated observability profile scrapes all three web containers by direct + target with a file-based Bearer credential, provisions a four-panel Grafana + dashboard, and routes the `up == 0` rule through Alertmanager. The canonical + drill observed both firing and resolved webhooks for the exact stopped + replica, restored all three targets, and left database counts unchanged. - The remaining rows above are still pending; this document is not a completion claim for the entire hardening goal. diff --git a/docs/operations.md b/docs/operations.md index 968b4a7..2ce0a7b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -134,17 +134,53 @@ integer-based; divide by `1_000_000` in PromQL when seconds are required. Definitions intentionally have no request path, user, request, or event-name labels that could create unbounded cardinality. -Prometheus itself, durable metrics retention, alert rules, notification -destinations, and measured alert thresholds are deployment responsibilities and -are not claimed by this repository. In Kubernetes, put the token in -`existingSecret`; configure the external scraper to send it as a Bearer token. - Metrics are local to each BEAM process. Discover and scrape every web pod or container as a distinct target and preserve Prometheus's `instance` label. A request through the load-balanced public route reaches only one replica and is therefore useful as an authorization/smoke check, not as a cluster-wide aggregate. +The isolated load project includes a local observability profile: + +```bash +./scripts/load-stack-up.sh +./scripts/observability-run.sh local-observability +``` + +The run script refuses the staging project, validates every current web +container's Compose labels, and writes a `file_sd` target for each observed +internal IP. Prometheus reads the Bearer value from a mode-`0600` runtime file, +not a tracked config or URL. Its direct request includes the internal +`X-Forwarded-Proto: https` signal required by the application's production SSL +rewrite while preserving the target's own `instance` label. + +Prometheus, Alertmanager, and Grafana are pinned by tag and digest. Their host +ports default to Docker-assigned values bound only to `127.0.0.1`; the run +prints the observed URLs. Grafana uses the random admin password generated in +ignored `.env.load`, disables anonymous signup, update checks, suggested plugin +installation, and its unused built-in alert engine. The Prometheus datasource +and four-panel dashboard are provisioned from tracked files. + +The verification stops exactly one scoped load web container. The +`WhoNeedHelpWebReplicaUnavailable` rule is based only on the factual +`up == 0` result; it is a local failure drill, not an invented latency, +capacity, or production SLO threshold. The script requires both firing and +resolved webhook payloads from Alertmanager, starts the same container, waits +for every direct target, and compares read-only database counts before and +after. Evidence is retained in `output/observability/` without the metrics or +Grafana secrets. + +Stop only the monitoring services with: + +```bash +./scripts/observability-stop.sh +``` + +Prometheus/Grafana/Alertmanager retention, production notification +destinations, production availability, and measured alert policies remain +deployment decisions. In Kubernetes, put the metrics token in +`existingSecret`; configure the external scraper to send it as a Bearer token. + ## Rollback boundary The release image is immutable and migrations run as a separate one-shot role. diff --git a/docs/performance.md b/docs/performance.md index 027fba0..77648db 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -204,6 +204,27 @@ Ignored evidence: - `output/resilience/compose-resilience-canonical-20260719f/` - `output/resilience/kind-rollout-canonical-20260719c/` +## Observed local monitoring drill + +The canonical observability run on 2026-07-19 generated one direct Prometheus +target for each of the three current load web containers. The exact expected +and observed `instance` lists matched, and all three targets were `up` before +the induced failure. Grafana reported a healthy Prometheus datasource and +served the tracked provisioned dashboard with four panels. + +The script then stopped the exact scoped container +`who_need_help_load-web-13`. Prometheus fired +`WhoNeedHelpWebReplicaUnavailable`, and Alertmanager delivered a webhook whose +labels identified that instance and the run. After the same container became +healthy again, all three direct targets returned to `up`, the alert cleared, +and a resolved webhook with the same instance/run labels arrived. The +application-table count diff was empty. These are observed local protocol +results, not a production monitoring or notification guarantee. + +Ignored evidence: + +- `output/observability/observability-canonical-20260719c/` + Stop the isolated containers without deleting their database volume: ```sh diff --git a/docs/verification.md b/docs/verification.md index 08d6442..1316790 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -20,6 +20,7 @@ results from product limits and unknown production properties. | Voluntary thanks | Implemented as an external optional link | A helper can expose an optional link after completion; the UI states that the platform does not process the payment. | The platform does not provide payments, escrow, refunds, tax reporting, or payment guarantees. | | Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. | | Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. | +| Local observability | Implemented and protocol-verified | Pinned Prometheus scraped all 3 direct load web targets with a file Bearer credential; Grafana provisioned a healthy datasource and four-panel dashboard; Alertmanager delivered firing and resolved webhooks for an induced scoped replica stop. | Local delivery does not establish production retention, notification-provider reliability, on-call policy, or measured alert thresholds. | ## Reproducible checks @@ -30,7 +31,7 @@ results from product limits and unknown production properties. - `mix compile --force --warnings-as-errors` and `mix format --check-formatted`: passed against the same final source. - `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning - threshold, actionlint 1.7.12, all four Compose renders, Helm lint, Trivy + threshold, actionlint 1.7.12, all five Compose renders, Helm lint, Trivy source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the production-image vulnerability scan. The rendered Helm manifest and Debian @@ -78,6 +79,17 @@ results from product limits and unknown production properties. PubSub, and left the database-count diff empty. All 305 readiness samples ultimately returned 200; two samples needed three total reconnect attempts during local single-node NodePort endpoint replacement. +- The canonical local observability drill matched all 3 generated/active + Prometheus instance targets, checked the provisioned Grafana datasource and + four-panel dashboard, received firing and resolved Alertmanager webhooks for + the exact stopped/recovered web replica, restored every target to `up`, and + left the application-table count diff empty. Its retained evidence contains + neither the metrics token nor the random Grafana password. +- The observability stop command changed only the four scoped monitoring + container states. All app/database/worker container IDs stayed unchanged and + running, and the checked user/request/message counts were identical before + and after. A subsequent full drill returned all monitoring services to + healthy. - The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium specs against a fresh PostGIS volume with two web and two worker replicas on 2026-07-19. The retained successful-run artifact directory is diff --git a/ops/observability/alertmanager.yml b/ops/observability/alertmanager.yml new file mode 100644 index 0000000..967c3ea --- /dev/null +++ b/ops/observability/alertmanager.yml @@ -0,0 +1,15 @@ +global: + resolve_timeout: 1m + +route: + receiver: local-webhook + group_by: [alertname, instance, run_id] + group_wait: 0s + group_interval: 1s + repeat_interval: 1h + +receivers: + - name: local-webhook + webhook_configs: + - url: http://alert-receiver:8080/alerts + send_resolved: true diff --git a/ops/observability/grafana/dashboards/who-need-help-overview.json b/ops/observability/grafana/dashboards/who-need-help-overview.json new file mode 100644 index 0000000..a7fd7f0 --- /dev/null +++ b/ops/observability/grafana/dashboards/who-need-help-overview.json @@ -0,0 +1,193 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "wnh-prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(up{job=\"who-need-help-web\"})", + "legendFormat": "available", + "range": true, + "refId": "A" + } + ], + "title": "Available web replicas", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "wnh-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(who_need_help_http_requests_total[1m])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP request rate by replica", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "wnh-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "who_need_help_vm_memory_total_bytes", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "BEAM memory by replica", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "wnh-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(who_need_help_database_queries_total[1m])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database query rate by replica", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 42, + "tags": ["who-need-help", "local"], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Who Need Help local overview", + "uid": "wnh-overview", + "version": 1 +} diff --git a/ops/observability/grafana/provisioning/dashboards/who-need-help.yml b/ops/observability/grafana/provisioning/dashboards/who-need-help.yml new file mode 100644 index 0000000..768ab78 --- /dev/null +++ b/ops/observability/grafana/provisioning/dashboards/who-need-help.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: Who Need Help + orgId: 1 + folder: Who Need Help + folderUid: wnh-local + type: file + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/ops/observability/grafana/provisioning/datasources/prometheus.yml b/ops/observability/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..803abc6 --- /dev/null +++ b/ops/observability/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,13 @@ +apiVersion: 1 +prune: true + +datasources: + - name: Who Need Help Prometheus + uid: wnh-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + prometheusType: Prometheus diff --git a/ops/observability/prometheus.template.yml b/ops/observability/prometheus.template.yml new file mode 100644 index 0000000..243a379 --- /dev/null +++ b/ops/observability/prometheus.template.yml @@ -0,0 +1,27 @@ +global: + scrape_interval: __SCRAPE_INTERVAL__ + evaluation_interval: __EVALUATION_INTERVAL__ + +rule_files: + - /etc/prometheus/rules.yml + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +scrape_configs: + - job_name: who-need-help-web + metrics_path: /metrics + authorization: + type: Bearer + credentials_file: /runtime/metrics-token + http_headers: + X-Forwarded-Proto: + values: + - https + file_sd_configs: + - files: + - /runtime/web-targets.json + refresh_interval: __SCRAPE_INTERVAL__ diff --git a/ops/observability/rules.yml b/ops/observability/rules.yml new file mode 100644 index 0000000..342e70d --- /dev/null +++ b/ops/observability/rules.yml @@ -0,0 +1,10 @@ +groups: + - name: who-need-help-local-availability + rules: + - alert: WhoNeedHelpWebReplicaUnavailable + expr: up{job="who-need-help-web"} == 0 + labels: + severity: local-drill + annotations: + summary: "The local Prometheus scraper cannot reach one web replica" + description: "The direct metrics target {{ $labels.instance }} is unavailable." diff --git a/scripts/alert-receiver.py b/scripts/alert-receiver.py new file mode 100755 index 0000000..7bc5ce1 --- /dev/null +++ b/scripts/alert-receiver.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Internal protocol-level webhook receiver for the local alert drill.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Lock +from typing import ClassVar + + +class AlertReceiver(BaseHTTPRequestHandler): + events: ClassVar[list[dict[str, object]]] = [] + events_lock: ClassVar[Lock] = Lock() + + def do_GET(self) -> None: + if self.path == "/healthz": + self._send_json(HTTPStatus.OK, {"status": "ok"}) + return + + if self.path == "/events": + with self.events_lock: + snapshot = list(self.events) + + self._send_json(HTTPStatus.OK, snapshot) + return + + self._send_json(HTTPStatus.NOT_FOUND, {"error": "not_found"}) + + def do_POST(self) -> None: + if self.path != "/alerts": + self._send_json(HTTPStatus.NOT_FOUND, {"error": "not_found"}) + return + + try: + content_length = int(self.headers.get("Content-Length", "")) + raw_payload = self.rfile.read(content_length) + payload = json.loads(raw_payload) + except (TypeError, ValueError, json.JSONDecodeError): + self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid_json"}) + return + + if not isinstance(payload, dict) or not isinstance(payload.get("alerts"), list): + self._send_json(HTTPStatus.UNPROCESSABLE_ENTITY, {"error": "invalid_alert_payload"}) + return + + event = { + "received_at": datetime.now(UTC).isoformat(), + "payload": payload, + } + + with self.events_lock: + self.events.append(event) + + self.send_response(HTTPStatus.NO_CONTENT) + self.end_headers() + + def log_message(self, message: str, *args: object) -> None: + print( + f"{self.log_date_time_string()} {self.client_address[0]} " + f"{message % args}", + flush=True, + ) + + def _send_json(self, status: HTTPStatus, payload: object) -> None: + body = json.dumps(payload, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", 8080), AlertReceiver).serve_forever() diff --git a/scripts/ensure-local-load-env.sh b/scripts/ensure-local-load-env.sh index d777507..4f72e94 100755 --- a/scripts/ensure-local-load-env.sh +++ b/scripts/ensure-local-load-env.sh @@ -20,6 +20,14 @@ if [ -f "$ENV_FILE" ]; then needs_resilience_interval=true needs_resilience_request_timeout=true needs_traefik_retry_attempts=true + needs_observability_prometheus_port=true + needs_observability_alertmanager_port=true + needs_observability_grafana_port=true + needs_observability_scrape_interval=true + needs_observability_evaluation_interval=true + needs_observability_timeout=true + needs_observability_grafana_user=true + needs_observability_grafana_password=true grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" && @@ -29,23 +37,52 @@ if [ -f "$ENV_FILE" ]; then grep -q '^LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=' "$ENV_FILE" && needs_resilience_request_timeout=false grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$ENV_FILE" && needs_traefik_retry_attempts=false + grep -q '^OBSERVABILITY_PROMETHEUS_PORT=' "$ENV_FILE" && + needs_observability_prometheus_port=false + grep -q '^OBSERVABILITY_ALERTMANAGER_PORT=' "$ENV_FILE" && + needs_observability_alertmanager_port=false + grep -q '^OBSERVABILITY_GRAFANA_PORT=' "$ENV_FILE" && + needs_observability_grafana_port=false + grep -q '^OBSERVABILITY_SCRAPE_INTERVAL=' "$ENV_FILE" && + needs_observability_scrape_interval=false + grep -q '^OBSERVABILITY_EVALUATION_INTERVAL=' "$ENV_FILE" && + needs_observability_evaluation_interval=false + grep -q '^OBSERVABILITY_TIMEOUT_SECONDS=' "$ENV_FILE" && + needs_observability_timeout=false + grep -q '^OBSERVABILITY_GRAFANA_ADMIN_USER=' "$ENV_FILE" && + needs_observability_grafana_user=false + grep -q '^OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=' "$ENV_FILE" && + needs_observability_grafana_password=false if [ "$needs_fixture_password" = false ] && [ "$needs_resilience_timeout" = false ] && [ "$needs_resilience_interval" = false ] && [ "$needs_resilience_request_timeout" = false ] && - [ "$needs_traefik_retry_attempts" = false ]; then + [ "$needs_traefik_retry_attempts" = false ] && + [ "$needs_observability_prometheus_port" = false ] && + [ "$needs_observability_alertmanager_port" = false ] && + [ "$needs_observability_grafana_port" = false ] && + [ "$needs_observability_scrape_interval" = false ] && + [ "$needs_observability_evaluation_interval" = false ] && + [ "$needs_observability_timeout" = false ] && + [ "$needs_observability_grafana_user" = false ] && + [ "$needs_observability_grafana_password" = false ]; then echo ".env.load already exists; no secret or experiment input was changed." exit 0 fi umask 077 load_fixture_password= + observability_grafana_admin_password= if [ "$needs_fixture_password" = true ]; then load_fixture_password=$(openssl rand -hex 24) fi + if [ "$needs_observability_grafana_password" = true ]; then + observability_grafana_admin_password=$(openssl rand -hex 32) + fi + { if [ "$needs_fixture_password" = true ]; then printf '\n# Added by the authenticated-load profile upgrade.\n' @@ -75,12 +112,61 @@ if [ -f "$ENV_FILE" ]; then if [ "$needs_traefik_retry_attempts" = true ]; then printf 'TRAEFIK_RETRY_ATTEMPTS=3\n' fi + + if [ "$needs_observability_prometheus_port" = true ] || + [ "$needs_observability_alertmanager_port" = true ] || + [ "$needs_observability_grafana_port" = true ] || + [ "$needs_observability_scrape_interval" = true ] || + [ "$needs_observability_evaluation_interval" = true ] || + [ "$needs_observability_timeout" = true ] || + [ "$needs_observability_grafana_user" = true ] || + [ "$needs_observability_grafana_password" = true ]; then + printf '\n# Added by the local observability-profile upgrade.\n' + fi + + if [ "$needs_observability_prometheus_port" = true ]; then + printf 'OBSERVABILITY_PROMETHEUS_PORT=0\n' + fi + + if [ "$needs_observability_alertmanager_port" = true ]; then + printf 'OBSERVABILITY_ALERTMANAGER_PORT=0\n' + fi + + if [ "$needs_observability_grafana_port" = true ]; then + printf 'OBSERVABILITY_GRAFANA_PORT=0\n' + fi + + if [ "$needs_observability_scrape_interval" = true ]; then + printf 'OBSERVABILITY_SCRAPE_INTERVAL=1s\n' + fi + + if [ "$needs_observability_evaluation_interval" = true ]; then + printf 'OBSERVABILITY_EVALUATION_INTERVAL=1s\n' + fi + + if [ "$needs_observability_timeout" = true ]; then + printf 'OBSERVABILITY_TIMEOUT_SECONDS=90\n' + fi + + if [ "$needs_observability_grafana_user" = true ]; then + printf 'OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin\n' + fi + + if [ "$needs_observability_grafana_password" = true ]; then + printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' \ + "$observability_grafana_admin_password" + fi } >>"$ENV_FILE" chmod 600 "$ENV_FILE" - unset load_fixture_password needs_fixture_password needs_resilience_timeout \ + unset load_fixture_password observability_grafana_admin_password \ + needs_fixture_password needs_resilience_timeout \ needs_resilience_interval needs_resilience_request_timeout \ - needs_traefik_retry_attempts - echo "Added missing authenticated-load/resilience inputs to ignored .env.load." + needs_traefik_retry_attempts needs_observability_prometheus_port \ + needs_observability_alertmanager_port needs_observability_grafana_port \ + needs_observability_scrape_interval needs_observability_evaluation_interval \ + needs_observability_timeout needs_observability_grafana_user \ + needs_observability_grafana_password + echo "Added missing load/resilience/observability inputs to ignored .env.load." exit 0 fi @@ -96,6 +182,7 @@ handover_secret=$(openssl rand -hex 64) release_cookie=$(openssl rand -hex 64) metrics_token=$(openssl rand -hex 32) load_fixture_password=$(openssl rand -hex 24) +observability_grafana_admin_password=$(openssl rand -hex 32) database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load" temporary=$(mktemp "${ENV_FILE}.XXXXXX") trap 'rm -f "$temporary"' EXIT HUP INT TERM @@ -107,6 +194,7 @@ HANDOVER_SECRET_VALUE=$handover_secret \ RELEASE_COOKIE_VALUE=$release_cookie \ METRICS_TOKEN_VALUE=$metrics_token \ LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \ +OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password \ perl -0pe ' s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g; s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g; @@ -115,6 +203,7 @@ LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \ s/GENERATE_RELEASE_COOKIE/$ENV{RELEASE_COOKIE_VALUE}/g; s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g; s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g; + s/GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD/$ENV{OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE}/g; ' "$TEMPLATE" >"$temporary" if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then @@ -126,6 +215,6 @@ chmod 600 "$temporary" mv "$temporary" "$ENV_FILE" trap - EXIT HUP INT TERM unset postgres_password secret_key_base handover_secret release_cookie metrics_token \ - load_fixture_password database_url + load_fixture_password observability_grafana_admin_password database_url echo "Generated independent load-profile secrets in ignored .env.load." diff --git a/scripts/observability-run.sh b/scripts/observability-run.sh new file mode 100755 index 0000000..b21a877 --- /dev/null +++ b/scripts/observability-run.sh @@ -0,0 +1,573 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +ENV_FILE="$ROOT/.env.load" +LABEL=${1:-"observability-$(date -u +%Y%m%dT%H%M%SZ)"} + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2 + exit 1 +fi + +set -a +# shellcheck source=/dev/null +. "$ENV_FILE" +set +a + +for name in LOAD_PROJECT LOAD_WEB_REPLICAS POSTGRES_DB METRICS_TOKEN \ + OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \ + OBSERVABILITY_GRAFANA_PORT OBSERVABILITY_SCRAPE_INTERVAL \ + OBSERVABILITY_EVALUATION_INTERVAL OBSERVABILITY_TIMEOUT_SECONDS \ + OBSERVABILITY_GRAFANA_ADMIN_USER OBSERVABILITY_GRAFANA_ADMIN_PASSWORD; do + if [[ -z "${!name:-}" ]]; then + echo "$name is missing from .env.load" >&2 + exit 1 + fi +done + +if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then + echo "The observability drill must not use the staging Compose project." >&2 + exit 1 +fi + +if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2 + exit 1 +fi + +if [[ ! "$LOAD_WEB_REPLICAS" =~ ^[1-9][0-9]*$ ]]; then + echo "LOAD_WEB_REPLICAS must be a positive integer." >&2 + exit 1 +fi + +for name in OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \ + OBSERVABILITY_GRAFANA_PORT; do + if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then + echo "$name must be a TCP port number from 0 through 65535." >&2 + exit 1 + fi +done + +for name in OBSERVABILITY_SCRAPE_INTERVAL OBSERVABILITY_EVALUATION_INTERVAL; do + if [[ ! "${!name}" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then + echo "$name must be a positive Prometheus duration using ms, s, m, or h." >&2 + exit 1 + fi +done + +if [[ ! "$OBSERVABILITY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + echo "OBSERVABILITY_TIMEOUT_SECONDS must be a positive integer." >&2 + exit 1 +fi + +for command in curl docker jq sed; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "Required command is unavailable: $command" >&2 + exit 1 + fi +done + +runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT" +output_dir="$ROOT/output/observability/$LABEL" +prometheus_runtime="$runtime_dir/prometheus" +grafana_runtime="$runtime_dir/grafana" +mkdir -p "$prometheus_runtime" "$grafana_runtime" "$output_dir" +chmod 700 "$ROOT/tmp" "$ROOT/tmp/observability" "$runtime_dir" \ + "$ROOT/output" "$ROOT/output/observability" "$output_dir" + +export OBSERVABILITY_RUNTIME_DIR="$runtime_dir" + +set_runtime_owner() { + local directory=$1 + local owner=$2 + + docker run --rm \ + --user 0:0 \ + --volume "$directory:/runtime" \ + --entrypoint /bin/sh \ + python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 \ + -euc "chown -R $owner /runtime; chmod 700 /runtime" +} + +host_owner="$(id -u):$(id -g)" +set_runtime_owner "$prometheus_runtime" "$host_owner" +set_runtime_owner "$grafana_runtime" "$host_owner" + +compose=( + docker compose + --env-file "$ENV_FILE" + -p "$LOAD_PROJECT" + -f compose.yaml + -f compose.load.yaml + -f compose.observability.yaml + --profile observability +) + +service_ids() { + "${compose[@]}" ps --all -q "$1" +} + +assert_scope() { + local container_id=$1 + local expected_service=$2 + local observed_project observed_service + + observed_project=$( + docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \ + "$container_id" + ) + observed_service=$( + docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \ + "$container_id" + ) + + if [[ "$observed_project" != "$LOAD_PROJECT" || + "$observed_service" != "$expected_service" ]]; then + echo "Container scope mismatch for $container_id." >&2 + exit 1 + fi +} + +wait_for_web() { + local container_id=$1 + local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + local state health + state=$(docker inspect --format '{{.State.Status}}' "$container_id") + health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \ + "$container_id") + + if [[ "$state" == "running" && "$health" == "healthy" ]]; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for web container $container_id to become healthy." >&2 + return 1 +} + +database_snapshot() { + # Variables are intentionally expanded inside the isolated PostGIS container. + # shellcheck disable=SC2016 + "${compose[@]}" exec -T db sh -c \ + 'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL' +BEGIN READ ONLY; +SELECT 'users' AS table_name, count(*) AS row_count FROM users +UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens +UNION ALL SELECT 'help_requests', count(*) FROM help_requests +UNION ALL SELECT 'messages', count(*) FROM messages +UNION ALL SELECT 'categories', count(*) FROM categories +UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments +UNION ALL SELECT 'activities', count(*) FROM activities +UNION ALL SELECT 'reports', count(*) FROM reports +UNION ALL SELECT 'social_identities', count(*) FROM social_identities +UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions +UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions +UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations +ORDER BY table_name; +COMMIT; +SQL +} + +published_url() { + local service=$1 + local target_port=$2 + local published + + published=$("${compose[@]}" port "$service" "$target_port" | head -n 1) + + if [[ ! "$published" =~ ^127[.]0[.]0[.]1:([1-9][0-9]*)$ ]]; then + echo "The $service port is not bound to an observed loopback port." >&2 + return 1 + fi + + printf 'http://%s' "$published" +} + +wait_for_http() { + local url=$1 + shift + local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + if curl --fail --silent --show-error "$@" "$url" >/dev/null 2>&1; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for $url." >&2 + return 1 +} + +fetch_receiver_events() { + local receiver_id + receiver_id=$(service_ids alert-receiver | head -n 1) + assert_scope "$receiver_id" alert-receiver + docker exec "$receiver_id" \ + python -c \ + 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/events", timeout=2).read().decode())' +} + +wait_for_targets_up() { + local expected=$1 + local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + if curl --fail --silent --show-error \ + "$prometheus_url/api/v1/targets?state=active" \ + >"$output_dir/targets-current.json" 2>/dev/null && + jq -e --argjson expected "$expected" --arg run_id "$LABEL" ' + [ + .data.activeTargets[] + | select( + .labels.job == "who-need-help-web" and + .labels.run_id == $run_id + ) + ] as $targets + | ($targets | length) == $expected + and all($targets[]; .health == "up") + ' "$output_dir/targets-current.json" >/dev/null; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for every direct web metrics target." >&2 + return 1 +} + +wait_for_firing_alert() { + local instance=$1 + local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \ + >"$output_dir/prometheus-alerts-firing.json" 2>/dev/null && + jq -e --arg instance "$instance" --arg run_id "$LABEL" ' + any( + .data.alerts[]; + .state == "firing" and + .labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and + .labels.instance == $instance and + .labels.run_id == $run_id + ) + ' "$output_dir/prometheus-alerts-firing.json" >/dev/null; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for the induced Prometheus alert." >&2 + return 1 +} + +wait_for_webhook_status() { + local instance=$1 + local expected_status=$2 + local destination=$3 + local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + if fetch_receiver_events >"$destination" 2>/dev/null && + jq -e \ + --arg instance "$instance" \ + --arg run_id "$LABEL" \ + --arg expected_status "$expected_status" ' + any( + .[].payload.alerts[]; + .status == $expected_status and + .labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and + .labels.instance == $instance and + .labels.run_id == $run_id + ) + ' "$destination" >/dev/null; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for the $expected_status Alertmanager webhook." >&2 + return 1 +} + +wait_for_alert_clear() { + local instance=$1 + local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \ + >"$output_dir/prometheus-alerts-resolved.json" 2>/dev/null && + jq -e --arg instance "$instance" --arg run_id "$LABEL" ' + all( + .data.alerts[]; + .labels.alertname != "WhoNeedHelpWebReplicaUnavailable" or + .labels.instance != $instance or + .labels.run_id != $run_id or + .state != "firing" + ) + ' "$output_dir/prometheus-alerts-resolved.json" >/dev/null; then + return 0 + fi + + sleep 1 + done + + echo "Timed out waiting for the induced Prometheus alert to resolve." >&2 + return 1 +} + +restore_web() { + if [[ -n "${drill_web_id:-}" ]]; then + local state + state=$(docker inspect --format '{{.State.Status}}' "$drill_web_id" 2>/dev/null || true) + + if [[ "$state" != "running" ]]; then + docker start "$drill_web_id" >/dev/null 2>&1 || true + fi + + wait_for_web "$drill_web_id" >/dev/null 2>&1 || true + fi +} + +cleanup() { + local status=$? + trap - EXIT HUP INT TERM + restore_web + exit "$status" +} + +trap cleanup EXIT HUP INT TERM + +mapfile -t web_ids < <(service_ids web) + +if [[ "${#web_ids[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then + echo "Expected $LOAD_WEB_REPLICAS running load web replicas; observed ${#web_ids[@]}." >&2 + exit 1 +fi + +internal_network=$( + docker network ls \ + --filter "label=com.docker.compose.project=$LOAD_PROJECT" \ + --filter "label=com.docker.compose.network=internal" \ + --format '{{.Name}}' +) + +if [[ -z "$internal_network" || "$internal_network" == *$'\n'* ]]; then + echo "Expected exactly one observed Compose internal network." >&2 + exit 1 +fi + +target_lines="$prometheus_runtime/web-targets.jsonl" +: >"$target_lines" + +for web_id in "${web_ids[@]}"; do + assert_scope "$web_id" web + wait_for_web "$web_id" + web_name=$(docker inspect --format '{{.Name}}' "$web_id") + web_name=${web_name#/} + web_ip=$( + docker inspect \ + --format "{{with index .NetworkSettings.Networks \"$internal_network\"}}{{.IPAddress}}{{end}}" \ + "$web_id" + ) + + if [[ ! "$web_ip" =~ ^[0-9]+([.][0-9]+){3}$ ]]; then + echo "No observed IPv4 address for $web_name on $internal_network." >&2 + exit 1 + fi + + jq -cn \ + --arg target "$web_ip:4000" \ + --arg instance "$web_name" \ + --arg compose_project "$LOAD_PROJECT" \ + --arg run_id "$LABEL" \ + '{ + targets: [$target], + labels: { + instance: $instance, + compose_project: $compose_project, + run_id: $run_id + } + }' >>"$target_lines" +done + +jq -s '.' "$target_lines" >"$prometheus_runtime/web-targets.json" +unlink "$target_lines" + +sed \ + -e "s/__SCRAPE_INTERVAL__/$OBSERVABILITY_SCRAPE_INTERVAL/g" \ + -e "s/__EVALUATION_INTERVAL__/$OBSERVABILITY_EVALUATION_INTERVAL/g" \ + "$ROOT/ops/observability/prometheus.template.yml" \ + >"$prometheus_runtime/prometheus.yml" +printf '%s' "$METRICS_TOKEN" >"$prometheus_runtime/metrics-token" +printf '%s' "$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \ + >"$grafana_runtime/admin-password" +cp "$prometheus_runtime/web-targets.json" "$output_dir/generated-targets.json" +chmod 600 "$prometheus_runtime/prometheus.yml" \ + "$prometheus_runtime/web-targets.json" \ + "$prometheus_runtime/metrics-token" \ + "$grafana_runtime/admin-password" \ + "$output_dir/generated-targets.json" +set_runtime_owner "$prometheus_runtime" "65534:65534" +set_runtime_owner "$grafana_runtime" "472:0" + +docker run --rm \ + --volume "$prometheus_runtime:/runtime:ro" \ + --volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \ + --entrypoint /bin/promtool \ + quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 \ + check config /runtime/prometheus.yml \ + >"$output_dir/promtool-check.txt" + +docker run --rm \ + --volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \ + --entrypoint /bin/amtool \ + quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d \ + check-config /etc/alertmanager/alertmanager.yml \ + >"$output_dir/amtool-check.txt" + +jq --exit-status 'type == "object" and .uid == "wnh-overview"' \ + "$ROOT/ops/observability/grafana/dashboards/who-need-help-overview.json" \ + >/dev/null + +for service in alert-receiver alertmanager prometheus grafana; do + while IFS= read -r existing_id; do + [[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service" + done < <(service_ids "$service") +done + +database_snapshot "$output_dir/database-before.txt" + +"${compose[@]}" up -d --force-recreate --wait \ + alert-receiver alertmanager prometheus grafana \ + >"$output_dir/compose-up.txt" + +for service in alert-receiver alertmanager prometheus grafana; do + service_id=$(service_ids "$service" | head -n 1) + assert_scope "$service_id" "$service" +done + +prometheus_url=$(published_url prometheus 9090) +alertmanager_url=$(published_url alertmanager 9093) +grafana_url=$(published_url grafana 3000) + +wait_for_http "$prometheus_url/-/ready" +wait_for_http "$alertmanager_url/-/ready" +wait_for_http "$grafana_url/api/health" +wait_for_http "$grafana_url/api/datasources/uid/wnh-prometheus/health" \ + --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" + +curl --fail --silent --show-error \ + --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \ + "$grafana_url/api/health" >"$output_dir/grafana-health.json" +curl --fail --silent --show-error \ + --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \ + "$grafana_url/api/datasources/uid/wnh-prometheus/health" \ + >"$output_dir/grafana-datasource-health.json" +curl --fail --silent --show-error \ + --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \ + "$grafana_url/api/dashboards/uid/wnh-overview" \ + >"$output_dir/grafana-dashboard.json" + +if ! jq -e ' + .dashboard.uid == "wnh-overview" and + .meta.provisioned == true and + (.dashboard.panels | length) == 4 +' "$output_dir/grafana-dashboard.json" >/dev/null; then + echo "The provisioned Grafana dashboard did not match the tracked dashboard." >&2 + exit 1 +fi + +wait_for_targets_up "$LOAD_WEB_REPLICAS" +mv "$output_dir/targets-current.json" "$output_dir/targets-before-drill.json" + +jq -n \ + --slurpfile expected "$output_dir/generated-targets.json" \ + --slurpfile observed "$output_dir/targets-before-drill.json" ' + { + expected_instances: ($expected[0] | map(.labels.instance) | sort), + observed_instances: ( + $observed[0].data.activeTargets + | map(select(.labels.job == "who-need-help-web") | .labels.instance) + | sort + ) + } + | . + { + exact_instance_match: (.expected_instances == .observed_instances) + } + ' >"$output_dir/target-summary.json" + +if ! jq -e '.exact_instance_match' "$output_dir/target-summary.json" >/dev/null; then + echo "Prometheus did not preserve every direct web instance target." >&2 + exit 1 +fi + +drill_web_id=${web_ids[0]} +assert_scope "$drill_web_id" web +drill_instance=$(docker inspect --format '{{.Name}}' "$drill_web_id") +drill_instance=${drill_instance#/} +docker stop --time 30 "$drill_web_id" >"$output_dir/stopped-web.txt" + +wait_for_firing_alert "$drill_instance" +wait_for_webhook_status \ + "$drill_instance" firing "$output_dir/alert-webhooks-firing.json" + +docker start "$drill_web_id" >"$output_dir/started-web.txt" +wait_for_web "$drill_web_id" +wait_for_targets_up "$LOAD_WEB_REPLICAS" +mv "$output_dir/targets-current.json" "$output_dir/targets-after-recovery.json" +wait_for_alert_clear "$drill_instance" +wait_for_webhook_status \ + "$drill_instance" resolved "$output_dir/alert-webhooks-complete.json" + +curl --fail --silent --show-error "$alertmanager_url/api/v2/alerts" \ + >"$output_dir/alertmanager-alerts-after-recovery.json" +database_snapshot "$output_dir/database-after.txt" + +if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \ + >"$output_dir/database-diff.txt"; then + echo "The observability drill changed tracked database counts." >&2 + exit 1 +fi + +"${compose[@]}" ps -a >"$output_dir/compose-after.txt" +"${compose[@]}" images --format json >"$output_dir/images.json" + +jq -n \ + --arg run_id "$LABEL" \ + --arg drill_instance "$drill_instance" \ + --arg prometheus_url "$prometheus_url" \ + --arg alertmanager_url "$alertmanager_url" \ + --arg grafana_url "$grafana_url" \ + --argjson web_replicas "$LOAD_WEB_REPLICAS" \ + '{ + run_id: $run_id, + web_replicas: $web_replicas, + direct_targets_up_before_and_after: true, + induced_instance: $drill_instance, + firing_webhook_observed: true, + resolved_webhook_observed: true, + database_count_diff_bytes: 0, + prometheus_url: $prometheus_url, + alertmanager_url: $alertmanager_url, + grafana_url: $grafana_url, + grafana_dashboard_path: "/d/wnh-overview/overview" + }' >"$output_dir/summary.json" + +trap - EXIT HUP INT TERM + +printf 'Observability evidence: %s\n' "$output_dir" +printf 'Prometheus: %s\nAlertmanager: %s\nGrafana: %s/d/wnh-overview/overview\n' \ + "$prometheus_url" "$alertmanager_url" "$grafana_url" +printf 'Grafana user: %s; its random password remains only in ignored .env.load.\n' \ + "$OBSERVABILITY_GRAFANA_ADMIN_USER" diff --git a/scripts/observability-stop.sh b/scripts/observability-stop.sh new file mode 100755 index 0000000..dcbd0f1 --- /dev/null +++ b/scripts/observability-stop.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +ENV_FILE="$ROOT/.env.load" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Missing $ENV_FILE; no observability project was selected." >&2 + exit 1 +fi + +set -a +# shellcheck source=/dev/null +. "$ENV_FILE" +set +a + +: "${LOAD_PROJECT:?LOAD_PROJECT is missing from .env.load}" + +if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then + echo "Refusing to stop services in the staging Compose project." >&2 + exit 1 +fi + +runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT" +export OBSERVABILITY_RUNTIME_DIR="$runtime_dir" + +compose=( + docker compose + --env-file "$ENV_FILE" + -p "$LOAD_PROJECT" + -f compose.yaml + -f compose.load.yaml + -f compose.observability.yaml + --profile observability +) + +for service in alert-receiver alertmanager prometheus grafana; do + while IFS= read -r container_id; do + [[ -z "$container_id" ]] && continue + observed_project=$( + docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \ + "$container_id" + ) + observed_service=$( + docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \ + "$container_id" + ) + + if [[ "$observed_project" != "$LOAD_PROJECT" || + "$observed_service" != "$service" ]]; then + echo "Container scope mismatch for $container_id." >&2 + exit 1 + fi + done < <("${compose[@]}" ps --all -q "$service") +done + +"${compose[@]}" stop grafana prometheus alertmanager alert-receiver + +echo "Stopped only the isolated observability services; app and metric volumes remain." diff --git a/scripts/quality.sh b/scripts/quality.sh index d0a2cdc..5c4ce46 100755 --- a/scripts/quality.sh +++ b/scripts/quality.sh @@ -8,6 +8,9 @@ SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.11.0@sha256:9955be09ea7f0dbf7ae9 HADOLINT_IMAGE="hadolint/hadolint:v2.14.0-debian@sha256:158cd0184dcaa18bd8ec20b61f4c1cabdf8b32a592d062f57bdcb8e4c1d312e2" ACTIONLINT_IMAGE="rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667" TRIVY_IMAGE="aquasec/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f" +PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893" +ALERTMANAGER_IMAGE="quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d" +PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4" run_id="$(date -u +%Y%m%d%H%M%S)-$$" project="wnh_quality_$(printf '%s' "$run_id" | tr -d '-')" @@ -64,8 +67,44 @@ E2E_OUTPUT_DIR="$scan_dir/e2e-output" docker compose --env-file .env.e2e.example -f compose.yaml -f compose.e2e.yaml config --quiet docker compose --env-file .env.load.example \ -f compose.yaml -f compose.load.yaml config --quiet +mkdir -p "$scan_dir/observability-runtime/prometheus" \ + "$scan_dir/observability-runtime/grafana" +OBSERVABILITY_RUNTIME_DIR="$scan_dir/observability-runtime" \ + docker compose --env-file .env.load.example \ + -f compose.yaml -f compose.load.yaml -f compose.observability.yaml \ + --profile observability config --quiet docker compose -p "$project" -f compose.quality.yaml config --quiet +echo "Validating local observability configuration" +sed \ + -e 's/__SCRAPE_INTERVAL__/1s/g' \ + -e 's/__EVALUATION_INTERVAL__/1s/g' \ + ops/observability/prometheus.template.yml \ + >"$scan_dir/observability-runtime/prometheus/prometheus.yml" +printf '%s' 'isolated-quality-metrics-token' \ + >"$scan_dir/observability-runtime/prometheus/metrics-token" +printf '%s\n' '[]' \ + >"$scan_dir/observability-runtime/prometheus/web-targets.json" +docker run --rm \ + --user 0:0 \ + --volume "$scan_dir/observability-runtime/prometheus:/runtime:ro" \ + --volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \ + --entrypoint /bin/promtool \ + "$PROMETHEUS_IMAGE" check config /runtime/prometheus.yml +docker run --rm \ + --user 0:0 \ + --volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \ + --entrypoint /bin/amtool \ + "$ALERTMANAGER_IMAGE" check-config /etc/alertmanager/alertmanager.yml +docker run --rm \ + --volume "$ROOT/scripts/alert-receiver.py:/src/alert-receiver.py:ro" \ + "$PYTHON_IMAGE" python -c \ + 'import py_compile; py_compile.compile("/src/alert-receiver.py", cfile="/tmp/alert-receiver.pyc", doraise=True)' +jq --exit-status \ + 'type == "object" and .uid == "wnh-overview" and (.panels | length) == 4' \ + ops/observability/grafana/dashboards/who-need-help-overview.json \ + >/dev/null + echo "Linting the Helm chart" "$ROOT/scripts/bootstrap-kubernetes-tools.sh" >/dev/null "$ROOT/.tools/bin/helm" lint \