feat: add verified local observability stack
This commit is contained in:
parent
594065deb8
commit
76a261b9ec
|
|
@ -59,3 +59,11 @@ LOAD_FIXTURE_PASSWORD=GENERATE_LOAD_FIXTURE_PASSWORD
|
||||||
LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=120
|
LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=120
|
||||||
LOAD_RESILIENCE_PROBE_INTERVAL_SECONDS=0.05
|
LOAD_RESILIENCE_PROBE_INTERVAL_SECONDS=0.05
|
||||||
LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=2
|
LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=2
|
||||||
|
OBSERVABILITY_PROMETHEUS_PORT=0
|
||||||
|
OBSERVABILITY_ALERTMANAGER_PORT=0
|
||||||
|
OBSERVABILITY_GRAFANA_PORT=0
|
||||||
|
OBSERVABILITY_SCRAPE_INTERVAL=1s
|
||||||
|
OBSERVABILITY_EVALUATION_INTERVAL=1s
|
||||||
|
OBSERVABILITY_TIMEOUT_SECONDS=90
|
||||||
|
OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin
|
||||||
|
OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD
|
||||||
|
|
|
||||||
21
README.md
21
README.md
|
|
@ -287,6 +287,27 @@ Compose crash/replacement and Oban retry checks use the separate load project:
|
||||||
Both scripts retain ignored evidence under `output/resilience/`; their exact
|
Both scripts retain ignored evidence under `output/resilience/`; their exact
|
||||||
mutation and cleanup boundaries are documented in the operations runbook.
|
mutation and cleanup boundaries are documented in the operations runbook.
|
||||||
|
|
||||||
|
Start and verify the local monitoring profile against every isolated load web
|
||||||
|
replica:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/observability-run.sh local-observability
|
||||||
|
```
|
||||||
|
|
||||||
|
The command validates Prometheus and Alertmanager configuration, provisions the
|
||||||
|
Grafana datasource and dashboard, discovers each current web container as a
|
||||||
|
separate target, and exercises a firing/resolved alert by stopping and
|
||||||
|
recovering exactly one verified load replica. It prints the loopback-only
|
||||||
|
random ports and retains non-secret evidence below `output/observability/`.
|
||||||
|
The generated Grafana password remains only in mode-`0600` `.env.load`.
|
||||||
|
|
||||||
|
Stop only the monitoring services while leaving their local metric volumes and
|
||||||
|
the load application running:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/observability-stop.sh
|
||||||
|
```
|
||||||
|
|
||||||
For an external cluster, provide a real PostgreSQL/PostGIS service and a
|
For an external cluster, provide a real PostgreSQL/PostGIS service and a
|
||||||
pre-created Secret through required `existingSecret`; the chart never renders
|
pre-created Secret through required `existingSecret`; the chart never renders
|
||||||
credentials from tracked values. The Secret must contain `DATABASE_URL`,
|
credentials from tracked values. The Secret must contain `DATABASE_URL`,
|
||||||
|
|
|
||||||
136
compose.observability.yaml
Normal file
136
compose.observability.yaml
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
services:
|
||||||
|
alert-receiver:
|
||||||
|
image: python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4
|
||||||
|
command: ["python", "/opt/who-need-help/alert-receiver.py"]
|
||||||
|
volumes:
|
||||||
|
- ./scripts/alert-receiver.py:/opt/who-need-help/alert-receiver.py:ro
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD",
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2).read()",
|
||||||
|
]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
user: "65532:65532"
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
|
cap_drop: [ALL]
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
networks: [internal]
|
||||||
|
restart: unless-stopped
|
||||||
|
profiles: [observability]
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
image: quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d
|
||||||
|
command:
|
||||||
|
- --config.file=/etc/alertmanager/alertmanager.yml
|
||||||
|
- --storage.path=/alertmanager
|
||||||
|
volumes:
|
||||||
|
- ./ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||||
|
- alertmanager_data:/alertmanager
|
||||||
|
ports:
|
||||||
|
- target: 9093
|
||||||
|
published: "${OBSERVABILITY_ALERTMANAGER_PORT:-0}"
|
||||||
|
host_ip: 127.0.0.1
|
||||||
|
protocol: tcp
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:9093/-/ready"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
depends_on:
|
||||||
|
alert-receiver:
|
||||||
|
condition: service_healthy
|
||||||
|
read_only: true
|
||||||
|
cap_drop: [ALL]
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
networks: [internal, observability_host]
|
||||||
|
restart: unless-stopped
|
||||||
|
profiles: [observability]
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893
|
||||||
|
command:
|
||||||
|
- --config.file=/runtime/prometheus.yml
|
||||||
|
- --storage.tsdb.path=/prometheus
|
||||||
|
volumes:
|
||||||
|
- ${OBSERVABILITY_RUNTIME_DIR:?Set OBSERVABILITY_RUNTIME_DIR for the observability profile}/prometheus:/runtime:ro
|
||||||
|
- ./ops/observability/rules.yml:/etc/prometheus/rules.yml:ro
|
||||||
|
- prometheus_data:/prometheus
|
||||||
|
ports:
|
||||||
|
- target: 9090
|
||||||
|
published: "${OBSERVABILITY_PROMETHEUS_PORT:-0}"
|
||||||
|
host_ip: 127.0.0.1
|
||||||
|
protocol: tcp
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:9090/-/ready"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
depends_on:
|
||||||
|
alertmanager:
|
||||||
|
condition: service_healthy
|
||||||
|
read_only: true
|
||||||
|
cap_drop: [ALL]
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
networks: [internal, observability_host]
|
||||||
|
restart: unless-stopped
|
||||||
|
profiles: [observability]
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63
|
||||||
|
environment:
|
||||||
|
GF_SECURITY_ADMIN_USER: ${OBSERVABILITY_GRAFANA_ADMIN_USER:?Set OBSERVABILITY_GRAFANA_ADMIN_USER}
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/wnh-secrets/admin-password
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||||
|
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||||
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||||
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||||
|
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||||
|
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||||
|
GF_UNIFIED_ALERTING_ENABLED: "false"
|
||||||
|
volumes:
|
||||||
|
- ${OBSERVABILITY_RUNTIME_DIR:?Set OBSERVABILITY_RUNTIME_DIR for the observability profile}/grafana:/run/wnh-secrets:ro
|
||||||
|
- ./ops/observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
|
||||||
|
- ./ops/observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
|
||||||
|
- ./ops/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
- grafana_data:/var/lib/grafana
|
||||||
|
ports:
|
||||||
|
- target: 3000
|
||||||
|
published: "${OBSERVABILITY_GRAFANA_PORT:-0}"
|
||||||
|
host_ip: 127.0.0.1
|
||||||
|
protocol: tcp
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "--spider", "--quiet", "http://127.0.0.1:3000/api/health"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
depends_on:
|
||||||
|
prometheus:
|
||||||
|
condition: service_healthy
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
|
- /var/log/grafana
|
||||||
|
cap_drop: [ALL]
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
networks: [internal, observability_host]
|
||||||
|
restart: unless-stopped
|
||||||
|
profiles: [observability]
|
||||||
|
|
||||||
|
networks:
|
||||||
|
observability_host:
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
alertmanager_data:
|
||||||
|
prometheus_data:
|
||||||
|
grafana_data:
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Dependency baseline
|
# Dependency baseline
|
||||||
|
|
||||||
Verified on 2026-07-18. This is a point-in-time stable baseline, not a claim
|
Verified through 2026-07-19. This is a point-in-time stable baseline, not a claim
|
||||||
that future security updates or major-version migrations can be avoided.
|
that future security updates or major-version migrations can be avoided.
|
||||||
Application locks and OCI digests remain committed so the same revision can be
|
Application locks and OCI digests remain committed so the same revision can be
|
||||||
rebuilt deterministically.
|
rebuilt deterministically.
|
||||||
|
|
@ -38,6 +38,10 @@ package checksums are in `mix.lock` and `assets/package-lock.json`.
|
||||||
| Traefik | 3.7.8 |
|
| Traefik | 3.7.8 |
|
||||||
| Mailpit | 1.30.4 |
|
| Mailpit | 1.30.4 |
|
||||||
| k6 load generator | 2.1.0 |
|
| k6 load generator | 2.1.0 |
|
||||||
|
| Prometheus | 3.13.1 |
|
||||||
|
| Alertmanager | 0.33.1 |
|
||||||
|
| Grafana | 13.1.0 |
|
||||||
|
| Python alert-boundary runtime | 3.14.6 / Alpine 3.23 |
|
||||||
| Debian builder/runner snapshot | trixie-20260713-slim |
|
| Debian builder/runner snapshot | trixie-20260713-slim |
|
||||||
|
|
||||||
Every external Compose/kind service image and every Dockerfile base image is
|
Every external Compose/kind service image and every Dockerfile base image is
|
||||||
|
|
@ -81,9 +85,10 @@ because the official SDK channel identifies it as a QPR beta.
|
||||||
| Sobelow | 0.14.1 |
|
| Sobelow | 0.14.1 |
|
||||||
| GitHub checkout action | 7.0.0 / commit `9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0` |
|
| GitHub checkout action | 7.0.0 / commit `9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0` |
|
||||||
|
|
||||||
The four containerized command-line tools are pinned by both exact tag and OCI
|
The four lint/security command-line images and the three observability
|
||||||
digest in `scripts/quality.sh`. The checkout action is pinned by commit in the
|
validation images are pinned by both exact tag and OCI digest in
|
||||||
workflow. Credo, Dialyxir, and Sobelow are locked in `mix.lock`.
|
`scripts/quality.sh`. The checkout action is pinned by commit in the workflow.
|
||||||
|
Credo, Dialyxir, and Sobelow are locked in `mix.lock`.
|
||||||
|
|
||||||
## Primary version sources
|
## Primary version sources
|
||||||
|
|
||||||
|
|
@ -97,6 +102,10 @@ workflow. Credo, Dialyxir, and Sobelow are locked in `mix.lock`.
|
||||||
- [Traefik releases](https://github.com/traefik/traefik/releases)
|
- [Traefik releases](https://github.com/traefik/traefik/releases)
|
||||||
- [Mailpit releases](https://github.com/axllent/mailpit/releases)
|
- [Mailpit releases](https://github.com/axllent/mailpit/releases)
|
||||||
- [k6 releases](https://github.com/grafana/k6/releases)
|
- [k6 releases](https://github.com/grafana/k6/releases)
|
||||||
|
- [Prometheus releases](https://github.com/prometheus/prometheus/releases)
|
||||||
|
- [Alertmanager releases](https://github.com/prometheus/alertmanager/releases)
|
||||||
|
- [Grafana releases](https://github.com/grafana/grafana/releases)
|
||||||
|
- [Python releases](https://www.python.org/downloads/)
|
||||||
- [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes)
|
- [Android Gradle Plugin 9.3.0 release notes](https://developer.android.com/build/releases/agp-9-3-0-release-notes)
|
||||||
- [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk)
|
- [Android 17 SDK setup](https://developer.android.com/about/versions/17/setup-sdk)
|
||||||
- [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests)
|
- [Build instrumented tests](https://developer.android.com/training/testing/instrumented-tests)
|
||||||
|
|
@ -126,4 +135,5 @@ docker run --rm who-need-help:node-deps npm outdated --json
|
||||||
./scripts/e2e-run.sh
|
./scripts/e2e-run.sh
|
||||||
./scripts/android-build.sh
|
./scripts/android-build.sh
|
||||||
./scripts/android-instrumentation-test.sh
|
./scripts/android-instrumentation-test.sh
|
||||||
|
./scripts/observability-run.sh local-observability
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ item below unless the evidence column explicitly describes a local mock.
|
||||||
| Localization and accessibility | Completed locally: product copy and custom validation messages are extracted; EN/UK/RU catalogs and localized category descriptions/structured values are implemented | 508 default and 40 error messages are current; RU/UK have no empty/fuzzy entries; 163 backend tests and all 8 browser specs pass, including locale persistence, keyboard, axe, themes, responsive widths, and reconnect |
|
| Localization and accessibility | Completed locally: product copy and custom validation messages are extracted; EN/UK/RU catalogs and localized category descriptions/structured values are implemented | 508 default and 40 error messages are current; RU/UK have no empty/fuzzy entries; 163 backend tests and all 8 browser specs pass, including locale persistence, keyboard, axe, themes, responsive widths, and reconnect |
|
||||||
| Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset |
|
| Database scale | Core discovery/chat/moderation lists call unbounded `Repo.all()` | Cursor-bounded queries pass behavior tests and measured `EXPLAIN ANALYZE` checks on an isolated generated dataset |
|
||||||
| Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data |
|
| Load and resilience | Public/readiness/heartbeat k6 profile exists | Authenticated writes, chat, tracking, reconnect, rolling replacement, and worker retry profiles pass without touching staging data |
|
||||||
| Observability | Protected Prometheus text endpoint exists | Local Prometheus/Grafana/Alertmanager profile scrapes every replica and an induced isolated failure exercises alert delivery |
|
| Observability | Completed locally: protected per-process metrics feed a pinned Prometheus/Grafana/Alertmanager profile | All 3 direct web targets are up before/after the drill; a verified replica stop delivers firing and resolved webhooks; Grafana datasource/dashboard and an empty DB-count diff are retained |
|
||||||
| Backup | Validated local custom-format dump and restore drill exist | An encrypted artifact is uploaded to local S3-compatible MinIO and restored into a fresh database; corruption and interrupted-upload checks fail closed |
|
| Backup | Validated local custom-format dump and restore drill exist | An encrypted artifact is uploaded to local S3-compatible MinIO and restored into a fresh database; corruption and interrupted-upload checks fail closed |
|
||||||
| External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout |
|
| External boundaries | Mailpit and a fake GitHub strategy cover parts of SMTP/OAuth | Local protocol-level SMTP/OAuth mocks and the applicable push adapter boundary cover success, rejection, retry, replay, and timeout |
|
||||||
| Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits |
|
| Final regression | 163 Phoenix tests plus reproducible browser and Android device suites | Browser, Android, API, DB, WebSocket, backup, monitoring, failure, cleanup, docs, and clean Git are verified from the final commits |
|
||||||
|
|
@ -89,5 +89,10 @@ The goal remains open while any row lacks reproducible local evidence.
|
||||||
The local single-node NodePort needed three reconnect attempts across 305
|
The local single-node NodePort needed three reconnect attempts across 305
|
||||||
ultimately successful samples; this is recorded rather than presented as
|
ultimately successful samples; this is recorded rather than presented as
|
||||||
raw transport continuity.
|
raw transport continuity.
|
||||||
|
- The isolated observability profile scrapes all three web containers by direct
|
||||||
|
target with a file-based Bearer credential, provisions a four-panel Grafana
|
||||||
|
dashboard, and routes the `up == 0` rule through Alertmanager. The canonical
|
||||||
|
drill observed both firing and resolved webhooks for the exact stopped
|
||||||
|
replica, restored all three targets, and left database counts unchanged.
|
||||||
- The remaining rows above are still pending; this document is not a
|
- The remaining rows above are still pending; this document is not a
|
||||||
completion claim for the entire hardening goal.
|
completion claim for the entire hardening goal.
|
||||||
|
|
|
||||||
|
|
@ -134,17 +134,53 @@ integer-based; divide by `1_000_000` in PromQL when seconds are required.
|
||||||
Definitions intentionally have no request path, user, request, or event-name
|
Definitions intentionally have no request path, user, request, or event-name
|
||||||
labels that could create unbounded cardinality.
|
labels that could create unbounded cardinality.
|
||||||
|
|
||||||
Prometheus itself, durable metrics retention, alert rules, notification
|
|
||||||
destinations, and measured alert thresholds are deployment responsibilities and
|
|
||||||
are not claimed by this repository. In Kubernetes, put the token in
|
|
||||||
`existingSecret`; configure the external scraper to send it as a Bearer token.
|
|
||||||
|
|
||||||
Metrics are local to each BEAM process. Discover and scrape every web pod or
|
Metrics are local to each BEAM process. Discover and scrape every web pod or
|
||||||
container as a distinct target and preserve Prometheus's `instance` label. A
|
container as a distinct target and preserve Prometheus's `instance` label. A
|
||||||
request through the load-balanced public route reaches only one replica and is
|
request through the load-balanced public route reaches only one replica and is
|
||||||
therefore useful as an authorization/smoke check, not as a cluster-wide
|
therefore useful as an authorization/smoke check, not as a cluster-wide
|
||||||
aggregate.
|
aggregate.
|
||||||
|
|
||||||
|
The isolated load project includes a local observability profile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/load-stack-up.sh
|
||||||
|
./scripts/observability-run.sh local-observability
|
||||||
|
```
|
||||||
|
|
||||||
|
The run script refuses the staging project, validates every current web
|
||||||
|
container's Compose labels, and writes a `file_sd` target for each observed
|
||||||
|
internal IP. Prometheus reads the Bearer value from a mode-`0600` runtime file,
|
||||||
|
not a tracked config or URL. Its direct request includes the internal
|
||||||
|
`X-Forwarded-Proto: https` signal required by the application's production SSL
|
||||||
|
rewrite while preserving the target's own `instance` label.
|
||||||
|
|
||||||
|
Prometheus, Alertmanager, and Grafana are pinned by tag and digest. Their host
|
||||||
|
ports default to Docker-assigned values bound only to `127.0.0.1`; the run
|
||||||
|
prints the observed URLs. Grafana uses the random admin password generated in
|
||||||
|
ignored `.env.load`, disables anonymous signup, update checks, suggested plugin
|
||||||
|
installation, and its unused built-in alert engine. The Prometheus datasource
|
||||||
|
and four-panel dashboard are provisioned from tracked files.
|
||||||
|
|
||||||
|
The verification stops exactly one scoped load web container. The
|
||||||
|
`WhoNeedHelpWebReplicaUnavailable` rule is based only on the factual
|
||||||
|
`up == 0` result; it is a local failure drill, not an invented latency,
|
||||||
|
capacity, or production SLO threshold. The script requires both firing and
|
||||||
|
resolved webhook payloads from Alertmanager, starts the same container, waits
|
||||||
|
for every direct target, and compares read-only database counts before and
|
||||||
|
after. Evidence is retained in `output/observability/` without the metrics or
|
||||||
|
Grafana secrets.
|
||||||
|
|
||||||
|
Stop only the monitoring services with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/observability-stop.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Prometheus/Grafana/Alertmanager retention, production notification
|
||||||
|
destinations, production availability, and measured alert policies remain
|
||||||
|
deployment decisions. In Kubernetes, put the metrics token in
|
||||||
|
`existingSecret`; configure the external scraper to send it as a Bearer token.
|
||||||
|
|
||||||
## Rollback boundary
|
## Rollback boundary
|
||||||
|
|
||||||
The release image is immutable and migrations run as a separate one-shot role.
|
The release image is immutable and migrations run as a separate one-shot role.
|
||||||
|
|
|
||||||
|
|
@ -204,6 +204,27 @@ Ignored evidence:
|
||||||
- `output/resilience/compose-resilience-canonical-20260719f/`
|
- `output/resilience/compose-resilience-canonical-20260719f/`
|
||||||
- `output/resilience/kind-rollout-canonical-20260719c/`
|
- `output/resilience/kind-rollout-canonical-20260719c/`
|
||||||
|
|
||||||
|
## Observed local monitoring drill
|
||||||
|
|
||||||
|
The canonical observability run on 2026-07-19 generated one direct Prometheus
|
||||||
|
target for each of the three current load web containers. The exact expected
|
||||||
|
and observed `instance` lists matched, and all three targets were `up` before
|
||||||
|
the induced failure. Grafana reported a healthy Prometheus datasource and
|
||||||
|
served the tracked provisioned dashboard with four panels.
|
||||||
|
|
||||||
|
The script then stopped the exact scoped container
|
||||||
|
`who_need_help_load-web-13`. Prometheus fired
|
||||||
|
`WhoNeedHelpWebReplicaUnavailable`, and Alertmanager delivered a webhook whose
|
||||||
|
labels identified that instance and the run. After the same container became
|
||||||
|
healthy again, all three direct targets returned to `up`, the alert cleared,
|
||||||
|
and a resolved webhook with the same instance/run labels arrived. The
|
||||||
|
application-table count diff was empty. These are observed local protocol
|
||||||
|
results, not a production monitoring or notification guarantee.
|
||||||
|
|
||||||
|
Ignored evidence:
|
||||||
|
|
||||||
|
- `output/observability/observability-canonical-20260719c/`
|
||||||
|
|
||||||
Stop the isolated containers without deleting their database volume:
|
Stop the isolated containers without deleting their database volume:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ results from product limits and unknown production properties.
|
||||||
| Voluntary thanks | Implemented as an external optional link | A helper can expose an optional link after completion; the UI states that the platform does not process the payment. | The platform does not provide payments, escrow, refunds, tax reporting, or payment guarantees. |
|
| Voluntary thanks | Implemented as an external optional link | A helper can expose an optional link after completion; the UI states that the platform does not process the payment. | The platform does not provide payments, escrow, refunds, tax reporting, or payment guarantees. |
|
||||||
| Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. |
|
| Android client | Local and public-staging clients implemented and emulator-verified | The native packages `org.whoneedhelp.mobile.debug` and `org.whoneedhelp.mobile.staging` launch the same authenticated LiveView app. Public HTTPS login, map, two-way chat, permission prompts, minimized foreground-service location updates, notification Stop, deep-link routing, and server cleanup were exercised on API 37. | Production signing, Play Store publication, verified Android App Links, unattended/background-permission tracking, and iOS are not implemented. |
|
||||||
| Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. |
|
| Multiple web/worker instances | Implemented and locally failure/rollout-verified | The isolated Compose profile passed BEAM crashes and sequential replacement with 3 web/2 worker replicas, all five nodes joined, PubSub passed, and 743/743 readiness requests succeeded. The project-owned kind cluster replaced all 2 web/2 worker pod UIDs under `maxUnavailable=0`; all four replacement pods joined and PubSub passed. | Local PostGIS is a single instance. Production database HA, backups, and recovery are operator work and are not claimed complete. |
|
||||||
|
| Local observability | Implemented and protocol-verified | Pinned Prometheus scraped all 3 direct load web targets with a file Bearer credential; Grafana provisioned a healthy datasource and four-panel dashboard; Alertmanager delivered firing and resolved webhooks for an induced scoped replica stop. | Local delivery does not establish production retention, notification-provider reliability, on-call policy, or measured alert thresholds. |
|
||||||
|
|
||||||
## Reproducible checks
|
## Reproducible checks
|
||||||
|
|
||||||
|
|
@ -30,7 +31,7 @@ results from product limits and unknown production properties.
|
||||||
- `mix compile --force --warnings-as-errors` and
|
- `mix compile --force --warnings-as-errors` and
|
||||||
`mix format --check-formatted`: passed against the same final source.
|
`mix format --check-formatted`: passed against the same final source.
|
||||||
- `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning
|
- `./scripts/quality.sh` passed ShellCheck 0.11.0, Hadolint 2.14.0 at warning
|
||||||
threshold, actionlint 1.7.12, all four Compose renders, Helm lint, Trivy
|
threshold, actionlint 1.7.12, all five Compose renders, Helm lint, Trivy
|
||||||
source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow
|
source/rendered-manifest scanning, xref, Credo high-priority checks, Sobelow
|
||||||
strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the
|
strict/private checks, Hex audit, 163 Phoenix tests, both npm audits, and the
|
||||||
production-image vulnerability scan. The rendered Helm manifest and Debian
|
production-image vulnerability scan. The rendered Helm manifest and Debian
|
||||||
|
|
@ -78,6 +79,17 @@ results from product limits and unknown production properties.
|
||||||
PubSub, and left the database-count diff empty. All 305 readiness samples
|
PubSub, and left the database-count diff empty. All 305 readiness samples
|
||||||
ultimately returned 200; two samples needed three total reconnect attempts
|
ultimately returned 200; two samples needed three total reconnect attempts
|
||||||
during local single-node NodePort endpoint replacement.
|
during local single-node NodePort endpoint replacement.
|
||||||
|
- The canonical local observability drill matched all 3 generated/active
|
||||||
|
Prometheus instance targets, checked the provisioned Grafana datasource and
|
||||||
|
four-panel dashboard, received firing and resolved Alertmanager webhooks for
|
||||||
|
the exact stopped/recovered web replica, restored every target to `up`, and
|
||||||
|
left the application-table count diff empty. Its retained evidence contains
|
||||||
|
neither the metrics token nor the random Grafana password.
|
||||||
|
- The observability stop command changed only the four scoped monitoring
|
||||||
|
container states. All app/database/worker container IDs stayed unchanged and
|
||||||
|
running, and the checked user/request/message counts were identical before
|
||||||
|
and after. A subsequent full drill returned all monitoring services to
|
||||||
|
healthy.
|
||||||
- The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium
|
- The committed browser suite passed its 1/1 bootstrap and all 8/8 Chromium
|
||||||
specs against a fresh PostGIS volume with two web and two worker replicas on
|
specs against a fresh PostGIS volume with two web and two worker replicas on
|
||||||
2026-07-19. The retained successful-run artifact directory is
|
2026-07-19. The retained successful-run artifact directory is
|
||||||
|
|
|
||||||
15
ops/observability/alertmanager.yml
Normal file
15
ops/observability/alertmanager.yml
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
global:
|
||||||
|
resolve_timeout: 1m
|
||||||
|
|
||||||
|
route:
|
||||||
|
receiver: local-webhook
|
||||||
|
group_by: [alertname, instance, run_id]
|
||||||
|
group_wait: 0s
|
||||||
|
group_interval: 1s
|
||||||
|
repeat_interval: 1h
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
- name: local-webhook
|
||||||
|
webhook_configs:
|
||||||
|
- url: http://alert-receiver:8080/alerts
|
||||||
|
send_resolved: true
|
||||||
193
ops/observability/grafana/dashboards/who-need-help-overview.json
Normal file
193
ops/observability/grafana/dashboards/who-need-help-overview.json
Normal file
|
|
@ -0,0 +1,193 @@
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"editable": false,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "wnh-prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"decimals": 0,
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 8,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": ["lastNotNull"],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "sum(up{job=\"who-need-help-web\"})",
|
||||||
|
"legendFormat": "available",
|
||||||
|
"range": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Available web replicas",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "wnh-prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 16,
|
||||||
|
"x": 8,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "rate(who_need_help_http_requests_total[1m])",
|
||||||
|
"legendFormat": "{{instance}}",
|
||||||
|
"range": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "HTTP request rate by replica",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "wnh-prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "bytes"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "who_need_help_vm_memory_total_bytes",
|
||||||
|
"legendFormat": "{{instance}}",
|
||||||
|
"range": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "BEAM memory by replica",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "wnh-prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "ops"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "rate(who_need_help_database_queries_total[1m])",
|
||||||
|
"legendFormat": "{{instance}}",
|
||||||
|
"range": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Database query rate by replica",
|
||||||
|
"type": "timeseries"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "5s",
|
||||||
|
"schemaVersion": 42,
|
||||||
|
"tags": ["who-need-help", "local"],
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-15m",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "Who Need Help local overview",
|
||||||
|
"uid": "wnh-overview",
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: Who Need Help
|
||||||
|
orgId: 1
|
||||||
|
folder: Who Need Help
|
||||||
|
folderUid: wnh-local
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
allowUiUpdates: false
|
||||||
|
updateIntervalSeconds: 10
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
apiVersion: 1
|
||||||
|
prune: true
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Who Need Help Prometheus
|
||||||
|
uid: wnh-prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
prometheusType: Prometheus
|
||||||
27
ops/observability/prometheus.template.yml
Normal file
27
ops/observability/prometheus.template.yml
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
global:
|
||||||
|
scrape_interval: __SCRAPE_INTERVAL__
|
||||||
|
evaluation_interval: __EVALUATION_INTERVAL__
|
||||||
|
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/rules.yml
|
||||||
|
|
||||||
|
alerting:
|
||||||
|
alertmanagers:
|
||||||
|
- static_configs:
|
||||||
|
- targets:
|
||||||
|
- alertmanager:9093
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: who-need-help-web
|
||||||
|
metrics_path: /metrics
|
||||||
|
authorization:
|
||||||
|
type: Bearer
|
||||||
|
credentials_file: /runtime/metrics-token
|
||||||
|
http_headers:
|
||||||
|
X-Forwarded-Proto:
|
||||||
|
values:
|
||||||
|
- https
|
||||||
|
file_sd_configs:
|
||||||
|
- files:
|
||||||
|
- /runtime/web-targets.json
|
||||||
|
refresh_interval: __SCRAPE_INTERVAL__
|
||||||
10
ops/observability/rules.yml
Normal file
10
ops/observability/rules.yml
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
groups:
|
||||||
|
- name: who-need-help-local-availability
|
||||||
|
rules:
|
||||||
|
- alert: WhoNeedHelpWebReplicaUnavailable
|
||||||
|
expr: up{job="who-need-help-web"} == 0
|
||||||
|
labels:
|
||||||
|
severity: local-drill
|
||||||
|
annotations:
|
||||||
|
summary: "The local Prometheus scraper cannot reach one web replica"
|
||||||
|
description: "The direct metrics target {{ $labels.instance }} is unavailable."
|
||||||
77
scripts/alert-receiver.py
Executable file
77
scripts/alert-receiver.py
Executable file
|
|
@ -0,0 +1,77 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Internal protocol-level webhook receiver for the local alert drill."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from http import HTTPStatus
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from threading import Lock
|
||||||
|
from typing import ClassVar
|
||||||
|
|
||||||
|
|
||||||
|
class AlertReceiver(BaseHTTPRequestHandler):
|
||||||
|
events: ClassVar[list[dict[str, object]]] = []
|
||||||
|
events_lock: ClassVar[Lock] = Lock()
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/healthz":
|
||||||
|
self._send_json(HTTPStatus.OK, {"status": "ok"})
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.path == "/events":
|
||||||
|
with self.events_lock:
|
||||||
|
snapshot = list(self.events)
|
||||||
|
|
||||||
|
self._send_json(HTTPStatus.OK, snapshot)
|
||||||
|
return
|
||||||
|
|
||||||
|
self._send_json(HTTPStatus.NOT_FOUND, {"error": "not_found"})
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
if self.path != "/alerts":
|
||||||
|
self._send_json(HTTPStatus.NOT_FOUND, {"error": "not_found"})
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
content_length = int(self.headers.get("Content-Length", ""))
|
||||||
|
raw_payload = self.rfile.read(content_length)
|
||||||
|
payload = json.loads(raw_payload)
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid_json"})
|
||||||
|
return
|
||||||
|
|
||||||
|
if not isinstance(payload, dict) or not isinstance(payload.get("alerts"), list):
|
||||||
|
self._send_json(HTTPStatus.UNPROCESSABLE_ENTITY, {"error": "invalid_alert_payload"})
|
||||||
|
return
|
||||||
|
|
||||||
|
event = {
|
||||||
|
"received_at": datetime.now(UTC).isoformat(),
|
||||||
|
"payload": payload,
|
||||||
|
}
|
||||||
|
|
||||||
|
with self.events_lock:
|
||||||
|
self.events.append(event)
|
||||||
|
|
||||||
|
self.send_response(HTTPStatus.NO_CONTENT)
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def log_message(self, message: str, *args: object) -> None:
|
||||||
|
print(
|
||||||
|
f"{self.log_date_time_string()} {self.client_address[0]} "
|
||||||
|
f"{message % args}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _send_json(self, status: HTTPStatus, payload: object) -> None:
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
ThreadingHTTPServer(("0.0.0.0", 8080), AlertReceiver).serve_forever()
|
||||||
|
|
@ -20,6 +20,14 @@ if [ -f "$ENV_FILE" ]; then
|
||||||
needs_resilience_interval=true
|
needs_resilience_interval=true
|
||||||
needs_resilience_request_timeout=true
|
needs_resilience_request_timeout=true
|
||||||
needs_traefik_retry_attempts=true
|
needs_traefik_retry_attempts=true
|
||||||
|
needs_observability_prometheus_port=true
|
||||||
|
needs_observability_alertmanager_port=true
|
||||||
|
needs_observability_grafana_port=true
|
||||||
|
needs_observability_scrape_interval=true
|
||||||
|
needs_observability_evaluation_interval=true
|
||||||
|
needs_observability_timeout=true
|
||||||
|
needs_observability_grafana_user=true
|
||||||
|
needs_observability_grafana_password=true
|
||||||
|
|
||||||
grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false
|
grep -q '^LOAD_FIXTURE_PASSWORD=' "$ENV_FILE" && needs_fixture_password=false
|
||||||
grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" &&
|
grep -q '^LOAD_RESILIENCE_RECOVERY_TIMEOUT_SECONDS=' "$ENV_FILE" &&
|
||||||
|
|
@ -29,23 +37,52 @@ if [ -f "$ENV_FILE" ]; then
|
||||||
grep -q '^LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=' "$ENV_FILE" &&
|
grep -q '^LOAD_RESILIENCE_REQUEST_TIMEOUT_SECONDS=' "$ENV_FILE" &&
|
||||||
needs_resilience_request_timeout=false
|
needs_resilience_request_timeout=false
|
||||||
grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$ENV_FILE" && needs_traefik_retry_attempts=false
|
grep -q '^TRAEFIK_RETRY_ATTEMPTS=' "$ENV_FILE" && needs_traefik_retry_attempts=false
|
||||||
|
grep -q '^OBSERVABILITY_PROMETHEUS_PORT=' "$ENV_FILE" &&
|
||||||
|
needs_observability_prometheus_port=false
|
||||||
|
grep -q '^OBSERVABILITY_ALERTMANAGER_PORT=' "$ENV_FILE" &&
|
||||||
|
needs_observability_alertmanager_port=false
|
||||||
|
grep -q '^OBSERVABILITY_GRAFANA_PORT=' "$ENV_FILE" &&
|
||||||
|
needs_observability_grafana_port=false
|
||||||
|
grep -q '^OBSERVABILITY_SCRAPE_INTERVAL=' "$ENV_FILE" &&
|
||||||
|
needs_observability_scrape_interval=false
|
||||||
|
grep -q '^OBSERVABILITY_EVALUATION_INTERVAL=' "$ENV_FILE" &&
|
||||||
|
needs_observability_evaluation_interval=false
|
||||||
|
grep -q '^OBSERVABILITY_TIMEOUT_SECONDS=' "$ENV_FILE" &&
|
||||||
|
needs_observability_timeout=false
|
||||||
|
grep -q '^OBSERVABILITY_GRAFANA_ADMIN_USER=' "$ENV_FILE" &&
|
||||||
|
needs_observability_grafana_user=false
|
||||||
|
grep -q '^OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=' "$ENV_FILE" &&
|
||||||
|
needs_observability_grafana_password=false
|
||||||
|
|
||||||
if [ "$needs_fixture_password" = false ] &&
|
if [ "$needs_fixture_password" = false ] &&
|
||||||
[ "$needs_resilience_timeout" = false ] &&
|
[ "$needs_resilience_timeout" = false ] &&
|
||||||
[ "$needs_resilience_interval" = false ] &&
|
[ "$needs_resilience_interval" = false ] &&
|
||||||
[ "$needs_resilience_request_timeout" = false ] &&
|
[ "$needs_resilience_request_timeout" = false ] &&
|
||||||
[ "$needs_traefik_retry_attempts" = false ]; then
|
[ "$needs_traefik_retry_attempts" = false ] &&
|
||||||
|
[ "$needs_observability_prometheus_port" = false ] &&
|
||||||
|
[ "$needs_observability_alertmanager_port" = false ] &&
|
||||||
|
[ "$needs_observability_grafana_port" = false ] &&
|
||||||
|
[ "$needs_observability_scrape_interval" = false ] &&
|
||||||
|
[ "$needs_observability_evaluation_interval" = false ] &&
|
||||||
|
[ "$needs_observability_timeout" = false ] &&
|
||||||
|
[ "$needs_observability_grafana_user" = false ] &&
|
||||||
|
[ "$needs_observability_grafana_password" = false ]; then
|
||||||
echo ".env.load already exists; no secret or experiment input was changed."
|
echo ".env.load already exists; no secret or experiment input was changed."
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
umask 077
|
umask 077
|
||||||
load_fixture_password=
|
load_fixture_password=
|
||||||
|
observability_grafana_admin_password=
|
||||||
|
|
||||||
if [ "$needs_fixture_password" = true ]; then
|
if [ "$needs_fixture_password" = true ]; then
|
||||||
load_fixture_password=$(openssl rand -hex 24)
|
load_fixture_password=$(openssl rand -hex 24)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_grafana_password" = true ]; then
|
||||||
|
observability_grafana_admin_password=$(openssl rand -hex 32)
|
||||||
|
fi
|
||||||
|
|
||||||
{
|
{
|
||||||
if [ "$needs_fixture_password" = true ]; then
|
if [ "$needs_fixture_password" = true ]; then
|
||||||
printf '\n# Added by the authenticated-load profile upgrade.\n'
|
printf '\n# Added by the authenticated-load profile upgrade.\n'
|
||||||
|
|
@ -75,12 +112,61 @@ if [ -f "$ENV_FILE" ]; then
|
||||||
if [ "$needs_traefik_retry_attempts" = true ]; then
|
if [ "$needs_traefik_retry_attempts" = true ]; then
|
||||||
printf 'TRAEFIK_RETRY_ATTEMPTS=3\n'
|
printf 'TRAEFIK_RETRY_ATTEMPTS=3\n'
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_prometheus_port" = true ] ||
|
||||||
|
[ "$needs_observability_alertmanager_port" = true ] ||
|
||||||
|
[ "$needs_observability_grafana_port" = true ] ||
|
||||||
|
[ "$needs_observability_scrape_interval" = true ] ||
|
||||||
|
[ "$needs_observability_evaluation_interval" = true ] ||
|
||||||
|
[ "$needs_observability_timeout" = true ] ||
|
||||||
|
[ "$needs_observability_grafana_user" = true ] ||
|
||||||
|
[ "$needs_observability_grafana_password" = true ]; then
|
||||||
|
printf '\n# Added by the local observability-profile upgrade.\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_prometheus_port" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_PROMETHEUS_PORT=0\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_alertmanager_port" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_ALERTMANAGER_PORT=0\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_grafana_port" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_GRAFANA_PORT=0\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_scrape_interval" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_SCRAPE_INTERVAL=1s\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_evaluation_interval" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_EVALUATION_INTERVAL=1s\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_timeout" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_TIMEOUT_SECONDS=90\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_grafana_user" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_GRAFANA_ADMIN_USER=local-admin\n'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$needs_observability_grafana_password" = true ]; then
|
||||||
|
printf 'OBSERVABILITY_GRAFANA_ADMIN_PASSWORD=%s\n' \
|
||||||
|
"$observability_grafana_admin_password"
|
||||||
|
fi
|
||||||
} >>"$ENV_FILE"
|
} >>"$ENV_FILE"
|
||||||
chmod 600 "$ENV_FILE"
|
chmod 600 "$ENV_FILE"
|
||||||
unset load_fixture_password needs_fixture_password needs_resilience_timeout \
|
unset load_fixture_password observability_grafana_admin_password \
|
||||||
|
needs_fixture_password needs_resilience_timeout \
|
||||||
needs_resilience_interval needs_resilience_request_timeout \
|
needs_resilience_interval needs_resilience_request_timeout \
|
||||||
needs_traefik_retry_attempts
|
needs_traefik_retry_attempts needs_observability_prometheus_port \
|
||||||
echo "Added missing authenticated-load/resilience inputs to ignored .env.load."
|
needs_observability_alertmanager_port needs_observability_grafana_port \
|
||||||
|
needs_observability_scrape_interval needs_observability_evaluation_interval \
|
||||||
|
needs_observability_timeout needs_observability_grafana_user \
|
||||||
|
needs_observability_grafana_password
|
||||||
|
echo "Added missing load/resilience/observability inputs to ignored .env.load."
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
@ -96,6 +182,7 @@ handover_secret=$(openssl rand -hex 64)
|
||||||
release_cookie=$(openssl rand -hex 64)
|
release_cookie=$(openssl rand -hex 64)
|
||||||
metrics_token=$(openssl rand -hex 32)
|
metrics_token=$(openssl rand -hex 32)
|
||||||
load_fixture_password=$(openssl rand -hex 24)
|
load_fixture_password=$(openssl rand -hex 24)
|
||||||
|
observability_grafana_admin_password=$(openssl rand -hex 32)
|
||||||
database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load"
|
database_url="ecto://wnh_load:${postgres_password}@db/who_need_help_load"
|
||||||
temporary=$(mktemp "${ENV_FILE}.XXXXXX")
|
temporary=$(mktemp "${ENV_FILE}.XXXXXX")
|
||||||
trap 'rm -f "$temporary"' EXIT HUP INT TERM
|
trap 'rm -f "$temporary"' EXIT HUP INT TERM
|
||||||
|
|
@ -107,6 +194,7 @@ HANDOVER_SECRET_VALUE=$handover_secret \
|
||||||
RELEASE_COOKIE_VALUE=$release_cookie \
|
RELEASE_COOKIE_VALUE=$release_cookie \
|
||||||
METRICS_TOKEN_VALUE=$metrics_token \
|
METRICS_TOKEN_VALUE=$metrics_token \
|
||||||
LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \
|
LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \
|
||||||
|
OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE=$observability_grafana_admin_password \
|
||||||
perl -0pe '
|
perl -0pe '
|
||||||
s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g;
|
s/GENERATE_POSTGRES_PASSWORD/$ENV{POSTGRES_PASSWORD_VALUE}/g;
|
||||||
s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g;
|
s/GENERATE_DATABASE_URL/$ENV{DATABASE_URL_VALUE}/g;
|
||||||
|
|
@ -115,6 +203,7 @@ LOAD_FIXTURE_PASSWORD_VALUE=$load_fixture_password \
|
||||||
s/GENERATE_RELEASE_COOKIE/$ENV{RELEASE_COOKIE_VALUE}/g;
|
s/GENERATE_RELEASE_COOKIE/$ENV{RELEASE_COOKIE_VALUE}/g;
|
||||||
s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g;
|
s/GENERATE_METRICS_TOKEN/$ENV{METRICS_TOKEN_VALUE}/g;
|
||||||
s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g;
|
s/GENERATE_LOAD_FIXTURE_PASSWORD/$ENV{LOAD_FIXTURE_PASSWORD_VALUE}/g;
|
||||||
|
s/GENERATE_OBSERVABILITY_GRAFANA_ADMIN_PASSWORD/$ENV{OBSERVABILITY_GRAFANA_ADMIN_PASSWORD_VALUE}/g;
|
||||||
' "$TEMPLATE" >"$temporary"
|
' "$TEMPLATE" >"$temporary"
|
||||||
|
|
||||||
if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then
|
if grep -Eq '^[A-Z0-9_]+=GENERATE_' "$temporary"; then
|
||||||
|
|
@ -126,6 +215,6 @@ chmod 600 "$temporary"
|
||||||
mv "$temporary" "$ENV_FILE"
|
mv "$temporary" "$ENV_FILE"
|
||||||
trap - EXIT HUP INT TERM
|
trap - EXIT HUP INT TERM
|
||||||
unset postgres_password secret_key_base handover_secret release_cookie metrics_token \
|
unset postgres_password secret_key_base handover_secret release_cookie metrics_token \
|
||||||
load_fixture_password database_url
|
load_fixture_password observability_grafana_admin_password database_url
|
||||||
|
|
||||||
echo "Generated independent load-profile secrets in ignored .env.load."
|
echo "Generated independent load-profile secrets in ignored .env.load."
|
||||||
|
|
|
||||||
573
scripts/observability-run.sh
Executable file
573
scripts/observability-run.sh
Executable file
|
|
@ -0,0 +1,573 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
ENV_FILE="$ROOT/.env.load"
|
||||||
|
LABEL=${1:-"observability-$(date -u +%Y%m%dT%H%M%SZ)"}
|
||||||
|
|
||||||
|
if [[ ! -f "$ENV_FILE" ]]; then
|
||||||
|
echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -a
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
. "$ENV_FILE"
|
||||||
|
set +a
|
||||||
|
|
||||||
|
for name in LOAD_PROJECT LOAD_WEB_REPLICAS POSTGRES_DB METRICS_TOKEN \
|
||||||
|
OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \
|
||||||
|
OBSERVABILITY_GRAFANA_PORT OBSERVABILITY_SCRAPE_INTERVAL \
|
||||||
|
OBSERVABILITY_EVALUATION_INTERVAL OBSERVABILITY_TIMEOUT_SECONDS \
|
||||||
|
OBSERVABILITY_GRAFANA_ADMIN_USER OBSERVABILITY_GRAFANA_ADMIN_PASSWORD; do
|
||||||
|
if [[ -z "${!name:-}" ]]; then
|
||||||
|
echo "$name is missing from .env.load" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
|
||||||
|
echo "The observability drill must not use the staging Compose project." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
|
||||||
|
echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! "$LOAD_WEB_REPLICAS" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "LOAD_WEB_REPLICAS must be a positive integer." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for name in OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \
|
||||||
|
OBSERVABILITY_GRAFANA_PORT; do
|
||||||
|
if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then
|
||||||
|
echo "$name must be a TCP port number from 0 through 65535." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
for name in OBSERVABILITY_SCRAPE_INTERVAL OBSERVABILITY_EVALUATION_INTERVAL; do
|
||||||
|
if [[ ! "${!name}" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then
|
||||||
|
echo "$name must be a positive Prometheus duration using ms, s, m, or h." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ! "$OBSERVABILITY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "OBSERVABILITY_TIMEOUT_SECONDS must be a positive integer." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for command in curl docker jq sed; do
|
||||||
|
if ! command -v "$command" >/dev/null 2>&1; then
|
||||||
|
echo "Required command is unavailable: $command" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT"
|
||||||
|
output_dir="$ROOT/output/observability/$LABEL"
|
||||||
|
prometheus_runtime="$runtime_dir/prometheus"
|
||||||
|
grafana_runtime="$runtime_dir/grafana"
|
||||||
|
mkdir -p "$prometheus_runtime" "$grafana_runtime" "$output_dir"
|
||||||
|
chmod 700 "$ROOT/tmp" "$ROOT/tmp/observability" "$runtime_dir" \
|
||||||
|
"$ROOT/output" "$ROOT/output/observability" "$output_dir"
|
||||||
|
|
||||||
|
export OBSERVABILITY_RUNTIME_DIR="$runtime_dir"
|
||||||
|
|
||||||
|
set_runtime_owner() {
|
||||||
|
local directory=$1
|
||||||
|
local owner=$2
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
--user 0:0 \
|
||||||
|
--volume "$directory:/runtime" \
|
||||||
|
--entrypoint /bin/sh \
|
||||||
|
python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 \
|
||||||
|
-euc "chown -R $owner /runtime; chmod 700 /runtime"
|
||||||
|
}
|
||||||
|
|
||||||
|
host_owner="$(id -u):$(id -g)"
|
||||||
|
set_runtime_owner "$prometheus_runtime" "$host_owner"
|
||||||
|
set_runtime_owner "$grafana_runtime" "$host_owner"
|
||||||
|
|
||||||
|
compose=(
|
||||||
|
docker compose
|
||||||
|
--env-file "$ENV_FILE"
|
||||||
|
-p "$LOAD_PROJECT"
|
||||||
|
-f compose.yaml
|
||||||
|
-f compose.load.yaml
|
||||||
|
-f compose.observability.yaml
|
||||||
|
--profile observability
|
||||||
|
)
|
||||||
|
|
||||||
|
service_ids() {
|
||||||
|
"${compose[@]}" ps --all -q "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_scope() {
|
||||||
|
local container_id=$1
|
||||||
|
local expected_service=$2
|
||||||
|
local observed_project observed_service
|
||||||
|
|
||||||
|
observed_project=$(
|
||||||
|
docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
|
||||||
|
"$container_id"
|
||||||
|
)
|
||||||
|
observed_service=$(
|
||||||
|
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
|
||||||
|
"$container_id"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$observed_project" != "$LOAD_PROJECT" ||
|
||||||
|
"$observed_service" != "$expected_service" ]]; then
|
||||||
|
echo "Container scope mismatch for $container_id." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_web() {
|
||||||
|
local container_id=$1
|
||||||
|
local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
local state health
|
||||||
|
state=$(docker inspect --format '{{.State.Status}}' "$container_id")
|
||||||
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \
|
||||||
|
"$container_id")
|
||||||
|
|
||||||
|
if [[ "$state" == "running" && "$health" == "healthy" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Timed out waiting for web container $container_id to become healthy." >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
database_snapshot() {
|
||||||
|
# Variables are intentionally expanded inside the isolated PostGIS container.
|
||||||
|
# shellcheck disable=SC2016
|
||||||
|
"${compose[@]}" exec -T db sh -c \
|
||||||
|
'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
|
||||||
|
--username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL'
|
||||||
|
BEGIN READ ONLY;
|
||||||
|
SELECT 'users' AS table_name, count(*) AS row_count FROM users
|
||||||
|
UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
|
||||||
|
UNION ALL SELECT 'help_requests', count(*) FROM help_requests
|
||||||
|
UNION ALL SELECT 'messages', count(*) FROM messages
|
||||||
|
UNION ALL SELECT 'categories', count(*) FROM categories
|
||||||
|
UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
|
||||||
|
UNION ALL SELECT 'activities', count(*) FROM activities
|
||||||
|
UNION ALL SELECT 'reports', count(*) FROM reports
|
||||||
|
UNION ALL SELECT 'social_identities', count(*) FROM social_identities
|
||||||
|
UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
|
||||||
|
UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
|
||||||
|
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations
|
||||||
|
ORDER BY table_name;
|
||||||
|
COMMIT;
|
||||||
|
SQL
|
||||||
|
}
|
||||||
|
|
||||||
|
published_url() {
|
||||||
|
local service=$1
|
||||||
|
local target_port=$2
|
||||||
|
local published
|
||||||
|
|
||||||
|
published=$("${compose[@]}" port "$service" "$target_port" | head -n 1)
|
||||||
|
|
||||||
|
if [[ ! "$published" =~ ^127[.]0[.]0[.]1:([1-9][0-9]*)$ ]]; then
|
||||||
|
echo "The $service port is not bound to an observed loopback port." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'http://%s' "$published"
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_http() {
|
||||||
|
local url=$1
|
||||||
|
shift
|
||||||
|
local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
if curl --fail --silent --show-error "$@" "$url" >/dev/null 2>&1; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Timed out waiting for $url." >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch_receiver_events() {
|
||||||
|
local receiver_id
|
||||||
|
receiver_id=$(service_ids alert-receiver | head -n 1)
|
||||||
|
assert_scope "$receiver_id" alert-receiver
|
||||||
|
docker exec "$receiver_id" \
|
||||||
|
python -c \
|
||||||
|
'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/events", timeout=2).read().decode())'
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_targets_up() {
|
||||||
|
local expected=$1
|
||||||
|
local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
if curl --fail --silent --show-error \
|
||||||
|
"$prometheus_url/api/v1/targets?state=active" \
|
||||||
|
>"$output_dir/targets-current.json" 2>/dev/null &&
|
||||||
|
jq -e --argjson expected "$expected" --arg run_id "$LABEL" '
|
||||||
|
[
|
||||||
|
.data.activeTargets[]
|
||||||
|
| select(
|
||||||
|
.labels.job == "who-need-help-web" and
|
||||||
|
.labels.run_id == $run_id
|
||||||
|
)
|
||||||
|
] as $targets
|
||||||
|
| ($targets | length) == $expected
|
||||||
|
and all($targets[]; .health == "up")
|
||||||
|
' "$output_dir/targets-current.json" >/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Timed out waiting for every direct web metrics target." >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_firing_alert() {
|
||||||
|
local instance=$1
|
||||||
|
local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \
|
||||||
|
>"$output_dir/prometheus-alerts-firing.json" 2>/dev/null &&
|
||||||
|
jq -e --arg instance "$instance" --arg run_id "$LABEL" '
|
||||||
|
any(
|
||||||
|
.data.alerts[];
|
||||||
|
.state == "firing" and
|
||||||
|
.labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and
|
||||||
|
.labels.instance == $instance and
|
||||||
|
.labels.run_id == $run_id
|
||||||
|
)
|
||||||
|
' "$output_dir/prometheus-alerts-firing.json" >/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Timed out waiting for the induced Prometheus alert." >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_webhook_status() {
|
||||||
|
local instance=$1
|
||||||
|
local expected_status=$2
|
||||||
|
local destination=$3
|
||||||
|
local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
if fetch_receiver_events >"$destination" 2>/dev/null &&
|
||||||
|
jq -e \
|
||||||
|
--arg instance "$instance" \
|
||||||
|
--arg run_id "$LABEL" \
|
||||||
|
--arg expected_status "$expected_status" '
|
||||||
|
any(
|
||||||
|
.[].payload.alerts[];
|
||||||
|
.status == $expected_status and
|
||||||
|
.labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and
|
||||||
|
.labels.instance == $instance and
|
||||||
|
.labels.run_id == $run_id
|
||||||
|
)
|
||||||
|
' "$destination" >/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Timed out waiting for the $expected_status Alertmanager webhook." >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_alert_clear() {
|
||||||
|
local instance=$1
|
||||||
|
local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))
|
||||||
|
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \
|
||||||
|
>"$output_dir/prometheus-alerts-resolved.json" 2>/dev/null &&
|
||||||
|
jq -e --arg instance "$instance" --arg run_id "$LABEL" '
|
||||||
|
all(
|
||||||
|
.data.alerts[];
|
||||||
|
.labels.alertname != "WhoNeedHelpWebReplicaUnavailable" or
|
||||||
|
.labels.instance != $instance or
|
||||||
|
.labels.run_id != $run_id or
|
||||||
|
.state != "firing"
|
||||||
|
)
|
||||||
|
' "$output_dir/prometheus-alerts-resolved.json" >/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Timed out waiting for the induced Prometheus alert to resolve." >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
restore_web() {
|
||||||
|
if [[ -n "${drill_web_id:-}" ]]; then
|
||||||
|
local state
|
||||||
|
state=$(docker inspect --format '{{.State.Status}}' "$drill_web_id" 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [[ "$state" != "running" ]]; then
|
||||||
|
docker start "$drill_web_id" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
wait_for_web "$drill_web_id" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local status=$?
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
restore_web
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT HUP INT TERM
|
||||||
|
|
||||||
|
mapfile -t web_ids < <(service_ids web)
|
||||||
|
|
||||||
|
if [[ "${#web_ids[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then
|
||||||
|
echo "Expected $LOAD_WEB_REPLICAS running load web replicas; observed ${#web_ids[@]}." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
internal_network=$(
|
||||||
|
docker network ls \
|
||||||
|
--filter "label=com.docker.compose.project=$LOAD_PROJECT" \
|
||||||
|
--filter "label=com.docker.compose.network=internal" \
|
||||||
|
--format '{{.Name}}'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ -z "$internal_network" || "$internal_network" == *$'\n'* ]]; then
|
||||||
|
echo "Expected exactly one observed Compose internal network." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
target_lines="$prometheus_runtime/web-targets.jsonl"
|
||||||
|
: >"$target_lines"
|
||||||
|
|
||||||
|
for web_id in "${web_ids[@]}"; do
|
||||||
|
assert_scope "$web_id" web
|
||||||
|
wait_for_web "$web_id"
|
||||||
|
web_name=$(docker inspect --format '{{.Name}}' "$web_id")
|
||||||
|
web_name=${web_name#/}
|
||||||
|
web_ip=$(
|
||||||
|
docker inspect \
|
||||||
|
--format "{{with index .NetworkSettings.Networks \"$internal_network\"}}{{.IPAddress}}{{end}}" \
|
||||||
|
"$web_id"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ ! "$web_ip" =~ ^[0-9]+([.][0-9]+){3}$ ]]; then
|
||||||
|
echo "No observed IPv4 address for $web_name on $internal_network." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
jq -cn \
|
||||||
|
--arg target "$web_ip:4000" \
|
||||||
|
--arg instance "$web_name" \
|
||||||
|
--arg compose_project "$LOAD_PROJECT" \
|
||||||
|
--arg run_id "$LABEL" \
|
||||||
|
'{
|
||||||
|
targets: [$target],
|
||||||
|
labels: {
|
||||||
|
instance: $instance,
|
||||||
|
compose_project: $compose_project,
|
||||||
|
run_id: $run_id
|
||||||
|
}
|
||||||
|
}' >>"$target_lines"
|
||||||
|
done
|
||||||
|
|
||||||
|
jq -s '.' "$target_lines" >"$prometheus_runtime/web-targets.json"
|
||||||
|
unlink "$target_lines"
|
||||||
|
|
||||||
|
sed \
|
||||||
|
-e "s/__SCRAPE_INTERVAL__/$OBSERVABILITY_SCRAPE_INTERVAL/g" \
|
||||||
|
-e "s/__EVALUATION_INTERVAL__/$OBSERVABILITY_EVALUATION_INTERVAL/g" \
|
||||||
|
"$ROOT/ops/observability/prometheus.template.yml" \
|
||||||
|
>"$prometheus_runtime/prometheus.yml"
|
||||||
|
printf '%s' "$METRICS_TOKEN" >"$prometheus_runtime/metrics-token"
|
||||||
|
printf '%s' "$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
|
||||||
|
>"$grafana_runtime/admin-password"
|
||||||
|
cp "$prometheus_runtime/web-targets.json" "$output_dir/generated-targets.json"
|
||||||
|
chmod 600 "$prometheus_runtime/prometheus.yml" \
|
||||||
|
"$prometheus_runtime/web-targets.json" \
|
||||||
|
"$prometheus_runtime/metrics-token" \
|
||||||
|
"$grafana_runtime/admin-password" \
|
||||||
|
"$output_dir/generated-targets.json"
|
||||||
|
set_runtime_owner "$prometheus_runtime" "65534:65534"
|
||||||
|
set_runtime_owner "$grafana_runtime" "472:0"
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
--volume "$prometheus_runtime:/runtime:ro" \
|
||||||
|
--volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \
|
||||||
|
--entrypoint /bin/promtool \
|
||||||
|
quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 \
|
||||||
|
check config /runtime/prometheus.yml \
|
||||||
|
>"$output_dir/promtool-check.txt"
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
--volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \
|
||||||
|
--entrypoint /bin/amtool \
|
||||||
|
quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d \
|
||||||
|
check-config /etc/alertmanager/alertmanager.yml \
|
||||||
|
>"$output_dir/amtool-check.txt"
|
||||||
|
|
||||||
|
jq --exit-status 'type == "object" and .uid == "wnh-overview"' \
|
||||||
|
"$ROOT/ops/observability/grafana/dashboards/who-need-help-overview.json" \
|
||||||
|
>/dev/null
|
||||||
|
|
||||||
|
for service in alert-receiver alertmanager prometheus grafana; do
|
||||||
|
while IFS= read -r existing_id; do
|
||||||
|
[[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service"
|
||||||
|
done < <(service_ids "$service")
|
||||||
|
done
|
||||||
|
|
||||||
|
database_snapshot "$output_dir/database-before.txt"
|
||||||
|
|
||||||
|
"${compose[@]}" up -d --force-recreate --wait \
|
||||||
|
alert-receiver alertmanager prometheus grafana \
|
||||||
|
>"$output_dir/compose-up.txt"
|
||||||
|
|
||||||
|
for service in alert-receiver alertmanager prometheus grafana; do
|
||||||
|
service_id=$(service_ids "$service" | head -n 1)
|
||||||
|
assert_scope "$service_id" "$service"
|
||||||
|
done
|
||||||
|
|
||||||
|
prometheus_url=$(published_url prometheus 9090)
|
||||||
|
alertmanager_url=$(published_url alertmanager 9093)
|
||||||
|
grafana_url=$(published_url grafana 3000)
|
||||||
|
|
||||||
|
wait_for_http "$prometheus_url/-/ready"
|
||||||
|
wait_for_http "$alertmanager_url/-/ready"
|
||||||
|
wait_for_http "$grafana_url/api/health"
|
||||||
|
wait_for_http "$grafana_url/api/datasources/uid/wnh-prometheus/health" \
|
||||||
|
--user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD"
|
||||||
|
|
||||||
|
curl --fail --silent --show-error \
|
||||||
|
--user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
|
||||||
|
"$grafana_url/api/health" >"$output_dir/grafana-health.json"
|
||||||
|
curl --fail --silent --show-error \
|
||||||
|
--user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
|
||||||
|
"$grafana_url/api/datasources/uid/wnh-prometheus/health" \
|
||||||
|
>"$output_dir/grafana-datasource-health.json"
|
||||||
|
curl --fail --silent --show-error \
|
||||||
|
--user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
|
||||||
|
"$grafana_url/api/dashboards/uid/wnh-overview" \
|
||||||
|
>"$output_dir/grafana-dashboard.json"
|
||||||
|
|
||||||
|
if ! jq -e '
|
||||||
|
.dashboard.uid == "wnh-overview" and
|
||||||
|
.meta.provisioned == true and
|
||||||
|
(.dashboard.panels | length) == 4
|
||||||
|
' "$output_dir/grafana-dashboard.json" >/dev/null; then
|
||||||
|
echo "The provisioned Grafana dashboard did not match the tracked dashboard." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
wait_for_targets_up "$LOAD_WEB_REPLICAS"
|
||||||
|
mv "$output_dir/targets-current.json" "$output_dir/targets-before-drill.json"
|
||||||
|
|
||||||
|
jq -n \
|
||||||
|
--slurpfile expected "$output_dir/generated-targets.json" \
|
||||||
|
--slurpfile observed "$output_dir/targets-before-drill.json" '
|
||||||
|
{
|
||||||
|
expected_instances: ($expected[0] | map(.labels.instance) | sort),
|
||||||
|
observed_instances: (
|
||||||
|
$observed[0].data.activeTargets
|
||||||
|
| map(select(.labels.job == "who-need-help-web") | .labels.instance)
|
||||||
|
| sort
|
||||||
|
)
|
||||||
|
}
|
||||||
|
| . + {
|
||||||
|
exact_instance_match: (.expected_instances == .observed_instances)
|
||||||
|
}
|
||||||
|
' >"$output_dir/target-summary.json"
|
||||||
|
|
||||||
|
if ! jq -e '.exact_instance_match' "$output_dir/target-summary.json" >/dev/null; then
|
||||||
|
echo "Prometheus did not preserve every direct web instance target." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
drill_web_id=${web_ids[0]}
|
||||||
|
assert_scope "$drill_web_id" web
|
||||||
|
drill_instance=$(docker inspect --format '{{.Name}}' "$drill_web_id")
|
||||||
|
drill_instance=${drill_instance#/}
|
||||||
|
docker stop --time 30 "$drill_web_id" >"$output_dir/stopped-web.txt"
|
||||||
|
|
||||||
|
wait_for_firing_alert "$drill_instance"
|
||||||
|
wait_for_webhook_status \
|
||||||
|
"$drill_instance" firing "$output_dir/alert-webhooks-firing.json"
|
||||||
|
|
||||||
|
docker start "$drill_web_id" >"$output_dir/started-web.txt"
|
||||||
|
wait_for_web "$drill_web_id"
|
||||||
|
wait_for_targets_up "$LOAD_WEB_REPLICAS"
|
||||||
|
mv "$output_dir/targets-current.json" "$output_dir/targets-after-recovery.json"
|
||||||
|
wait_for_alert_clear "$drill_instance"
|
||||||
|
wait_for_webhook_status \
|
||||||
|
"$drill_instance" resolved "$output_dir/alert-webhooks-complete.json"
|
||||||
|
|
||||||
|
curl --fail --silent --show-error "$alertmanager_url/api/v2/alerts" \
|
||||||
|
>"$output_dir/alertmanager-alerts-after-recovery.json"
|
||||||
|
database_snapshot "$output_dir/database-after.txt"
|
||||||
|
|
||||||
|
if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \
|
||||||
|
>"$output_dir/database-diff.txt"; then
|
||||||
|
echo "The observability drill changed tracked database counts." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
"${compose[@]}" ps -a >"$output_dir/compose-after.txt"
|
||||||
|
"${compose[@]}" images --format json >"$output_dir/images.json"
|
||||||
|
|
||||||
|
jq -n \
|
||||||
|
--arg run_id "$LABEL" \
|
||||||
|
--arg drill_instance "$drill_instance" \
|
||||||
|
--arg prometheus_url "$prometheus_url" \
|
||||||
|
--arg alertmanager_url "$alertmanager_url" \
|
||||||
|
--arg grafana_url "$grafana_url" \
|
||||||
|
--argjson web_replicas "$LOAD_WEB_REPLICAS" \
|
||||||
|
'{
|
||||||
|
run_id: $run_id,
|
||||||
|
web_replicas: $web_replicas,
|
||||||
|
direct_targets_up_before_and_after: true,
|
||||||
|
induced_instance: $drill_instance,
|
||||||
|
firing_webhook_observed: true,
|
||||||
|
resolved_webhook_observed: true,
|
||||||
|
database_count_diff_bytes: 0,
|
||||||
|
prometheus_url: $prometheus_url,
|
||||||
|
alertmanager_url: $alertmanager_url,
|
||||||
|
grafana_url: $grafana_url,
|
||||||
|
grafana_dashboard_path: "/d/wnh-overview/overview"
|
||||||
|
}' >"$output_dir/summary.json"
|
||||||
|
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
|
||||||
|
printf 'Observability evidence: %s\n' "$output_dir"
|
||||||
|
printf 'Prometheus: %s\nAlertmanager: %s\nGrafana: %s/d/wnh-overview/overview\n' \
|
||||||
|
"$prometheus_url" "$alertmanager_url" "$grafana_url"
|
||||||
|
printf 'Grafana user: %s; its random password remains only in ignored .env.load.\n' \
|
||||||
|
"$OBSERVABILITY_GRAFANA_ADMIN_USER"
|
||||||
59
scripts/observability-stop.sh
Executable file
59
scripts/observability-stop.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||||
|
ENV_FILE="$ROOT/.env.load"
|
||||||
|
|
||||||
|
if [[ ! -f "$ENV_FILE" ]]; then
|
||||||
|
echo "Missing $ENV_FILE; no observability project was selected." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -a
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
. "$ENV_FILE"
|
||||||
|
set +a
|
||||||
|
|
||||||
|
: "${LOAD_PROJECT:?LOAD_PROJECT is missing from .env.load}"
|
||||||
|
|
||||||
|
if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
|
||||||
|
echo "Refusing to stop services in the staging Compose project." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT"
|
||||||
|
export OBSERVABILITY_RUNTIME_DIR="$runtime_dir"
|
||||||
|
|
||||||
|
compose=(
|
||||||
|
docker compose
|
||||||
|
--env-file "$ENV_FILE"
|
||||||
|
-p "$LOAD_PROJECT"
|
||||||
|
-f compose.yaml
|
||||||
|
-f compose.load.yaml
|
||||||
|
-f compose.observability.yaml
|
||||||
|
--profile observability
|
||||||
|
)
|
||||||
|
|
||||||
|
for service in alert-receiver alertmanager prometheus grafana; do
|
||||||
|
while IFS= read -r container_id; do
|
||||||
|
[[ -z "$container_id" ]] && continue
|
||||||
|
observed_project=$(
|
||||||
|
docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
|
||||||
|
"$container_id"
|
||||||
|
)
|
||||||
|
observed_service=$(
|
||||||
|
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
|
||||||
|
"$container_id"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$observed_project" != "$LOAD_PROJECT" ||
|
||||||
|
"$observed_service" != "$service" ]]; then
|
||||||
|
echo "Container scope mismatch for $container_id." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done < <("${compose[@]}" ps --all -q "$service")
|
||||||
|
done
|
||||||
|
|
||||||
|
"${compose[@]}" stop grafana prometheus alertmanager alert-receiver
|
||||||
|
|
||||||
|
echo "Stopped only the isolated observability services; app and metric volumes remain."
|
||||||
|
|
@ -8,6 +8,9 @@ SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.11.0@sha256:9955be09ea7f0dbf7ae9
|
||||||
HADOLINT_IMAGE="hadolint/hadolint:v2.14.0-debian@sha256:158cd0184dcaa18bd8ec20b61f4c1cabdf8b32a592d062f57bdcb8e4c1d312e2"
|
HADOLINT_IMAGE="hadolint/hadolint:v2.14.0-debian@sha256:158cd0184dcaa18bd8ec20b61f4c1cabdf8b32a592d062f57bdcb8e4c1d312e2"
|
||||||
ACTIONLINT_IMAGE="rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667"
|
ACTIONLINT_IMAGE="rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667"
|
||||||
TRIVY_IMAGE="aquasec/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f"
|
TRIVY_IMAGE="aquasec/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f"
|
||||||
|
PROMETHEUS_IMAGE="quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893"
|
||||||
|
ALERTMANAGER_IMAGE="quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d"
|
||||||
|
PYTHON_IMAGE="python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4"
|
||||||
|
|
||||||
run_id="$(date -u +%Y%m%d%H%M%S)-$$"
|
run_id="$(date -u +%Y%m%d%H%M%S)-$$"
|
||||||
project="wnh_quality_$(printf '%s' "$run_id" | tr -d '-')"
|
project="wnh_quality_$(printf '%s' "$run_id" | tr -d '-')"
|
||||||
|
|
@ -64,8 +67,44 @@ E2E_OUTPUT_DIR="$scan_dir/e2e-output" docker compose --env-file .env.e2e.example
|
||||||
-f compose.yaml -f compose.e2e.yaml config --quiet
|
-f compose.yaml -f compose.e2e.yaml config --quiet
|
||||||
docker compose --env-file .env.load.example \
|
docker compose --env-file .env.load.example \
|
||||||
-f compose.yaml -f compose.load.yaml config --quiet
|
-f compose.yaml -f compose.load.yaml config --quiet
|
||||||
|
mkdir -p "$scan_dir/observability-runtime/prometheus" \
|
||||||
|
"$scan_dir/observability-runtime/grafana"
|
||||||
|
OBSERVABILITY_RUNTIME_DIR="$scan_dir/observability-runtime" \
|
||||||
|
docker compose --env-file .env.load.example \
|
||||||
|
-f compose.yaml -f compose.load.yaml -f compose.observability.yaml \
|
||||||
|
--profile observability config --quiet
|
||||||
docker compose -p "$project" -f compose.quality.yaml config --quiet
|
docker compose -p "$project" -f compose.quality.yaml config --quiet
|
||||||
|
|
||||||
|
echo "Validating local observability configuration"
|
||||||
|
sed \
|
||||||
|
-e 's/__SCRAPE_INTERVAL__/1s/g' \
|
||||||
|
-e 's/__EVALUATION_INTERVAL__/1s/g' \
|
||||||
|
ops/observability/prometheus.template.yml \
|
||||||
|
>"$scan_dir/observability-runtime/prometheus/prometheus.yml"
|
||||||
|
printf '%s' 'isolated-quality-metrics-token' \
|
||||||
|
>"$scan_dir/observability-runtime/prometheus/metrics-token"
|
||||||
|
printf '%s\n' '[]' \
|
||||||
|
>"$scan_dir/observability-runtime/prometheus/web-targets.json"
|
||||||
|
docker run --rm \
|
||||||
|
--user 0:0 \
|
||||||
|
--volume "$scan_dir/observability-runtime/prometheus:/runtime:ro" \
|
||||||
|
--volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \
|
||||||
|
--entrypoint /bin/promtool \
|
||||||
|
"$PROMETHEUS_IMAGE" check config /runtime/prometheus.yml
|
||||||
|
docker run --rm \
|
||||||
|
--user 0:0 \
|
||||||
|
--volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \
|
||||||
|
--entrypoint /bin/amtool \
|
||||||
|
"$ALERTMANAGER_IMAGE" check-config /etc/alertmanager/alertmanager.yml
|
||||||
|
docker run --rm \
|
||||||
|
--volume "$ROOT/scripts/alert-receiver.py:/src/alert-receiver.py:ro" \
|
||||||
|
"$PYTHON_IMAGE" python -c \
|
||||||
|
'import py_compile; py_compile.compile("/src/alert-receiver.py", cfile="/tmp/alert-receiver.pyc", doraise=True)'
|
||||||
|
jq --exit-status \
|
||||||
|
'type == "object" and .uid == "wnh-overview" and (.panels | length) == 4' \
|
||||||
|
ops/observability/grafana/dashboards/who-need-help-overview.json \
|
||||||
|
>/dev/null
|
||||||
|
|
||||||
echo "Linting the Helm chart"
|
echo "Linting the Helm chart"
|
||||||
"$ROOT/scripts/bootstrap-kubernetes-tools.sh" >/dev/null
|
"$ROOT/scripts/bootstrap-kubernetes-tools.sh" >/dev/null
|
||||||
"$ROOT/.tools/bin/helm" lint \
|
"$ROOT/.tools/bin/helm" lint \
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user