who_need_help/scripts/observability-run.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
ENV_FILE="$ROOT/.env.load"
LABEL=${1:-"observability-$(date -u +%Y%m%dT%H%M%SZ)"}

if [[ ! -f "$ENV_FILE" ]]; then
  echo "Missing $ENV_FILE. Run scripts/ensure-local-load-env.sh first." >&2
  exit 1
fi

set -a
# shellcheck source=/dev/null
. "$ENV_FILE"
set +a

for name in LOAD_PROJECT LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS POSTGRES_DB METRICS_TOKEN \
  OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \
  OBSERVABILITY_GRAFANA_PORT OBSERVABILITY_SCRAPE_INTERVAL \
  OBSERVABILITY_EVALUATION_INTERVAL OBSERVABILITY_TIMEOUT_SECONDS \
  OBSERVABILITY_GRAFANA_ADMIN_USER OBSERVABILITY_GRAFANA_ADMIN_PASSWORD; do
  if [[ -z "${!name:-}" ]]; then
    echo "$name is missing from .env.load" >&2
    exit 1
  fi
done

if [[ "$LOAD_PROJECT" == "who_need_help" ]]; then
  echo "The observability drill must not use the staging Compose project." >&2
  exit 1
fi

if [[ ! "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then
  echo "Run label may contain only letters, numbers, dot, underscore, and dash." >&2
  exit 1
fi

for name in LOAD_WEB_REPLICAS LOAD_WORKER_REPLICAS; do
  if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then
    echo "$name must be a positive integer." >&2
    exit 1
  fi
done

for name in OBSERVABILITY_PROMETHEUS_PORT OBSERVABILITY_ALERTMANAGER_PORT \
  OBSERVABILITY_GRAFANA_PORT; do
  if [[ ! "${!name}" =~ ^[0-9]+$ ]] || ((10#${!name} > 65535)); then
    echo "$name must be a TCP port number from 0 through 65535." >&2
    exit 1
  fi
done

for name in OBSERVABILITY_SCRAPE_INTERVAL OBSERVABILITY_EVALUATION_INTERVAL; do
  if [[ ! "${!name}" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then
    echo "$name must be a positive Prometheus duration using ms, s, m, or h." >&2
    exit 1
  fi
done

if [[ ! "$OBSERVABILITY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
  echo "OBSERVABILITY_TIMEOUT_SECONDS must be a positive integer." >&2
  exit 1
fi

for command in curl docker jq sed; do
  if ! command -v "$command" >/dev/null 2>&1; then
    echo "Required command is unavailable: $command" >&2
    exit 1
  fi
done

runtime_dir="$ROOT/tmp/observability/$LOAD_PROJECT"
output_dir="$ROOT/output/observability/$LABEL"
prometheus_runtime="$runtime_dir/prometheus"
grafana_runtime="$runtime_dir/grafana"
mkdir -p "$prometheus_runtime" "$grafana_runtime" "$output_dir"
chmod 700 "$ROOT/tmp" "$ROOT/tmp/observability" "$runtime_dir" \
  "$ROOT/output" "$ROOT/output/observability" "$output_dir"

export OBSERVABILITY_RUNTIME_DIR="$runtime_dir"

set_runtime_owner() {
  local directory=$1
  local owner=$2

  docker run --rm \
    --user 0:0 \
    --volume "$directory:/runtime" \
    --entrypoint /bin/sh \
    python:3.14.6-alpine3.23@sha256:b165067c5afc37fa5608a3c05609cc3d51aafd808a30fbfd822ee594fef55ad4 \
    -euc "chown -R $owner /runtime; chmod 700 /runtime"
}

host_owner="$(id -u):$(id -g)"
set_runtime_owner "$prometheus_runtime" "$host_owner"
set_runtime_owner "$grafana_runtime" "$host_owner"

compose=(
  docker compose
  --env-file "$ENV_FILE"
  -p "$LOAD_PROJECT"
  -f compose.yaml
  -f compose.load.yaml
  -f compose.observability.yaml
  --profile observability
)

service_ids() {
  "${compose[@]}" ps --all -q "$1"
}

assert_scope() {
  local container_id=$1
  local expected_service=$2
  local observed_project observed_service

  observed_project=$(
    docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' \
      "$container_id"
  )
  observed_service=$(
    docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}}' \
      "$container_id"
  )

  if [[ "$observed_project" != "$LOAD_PROJECT" ||
    "$observed_service" != "$expected_service" ]]; then
    echo "Container scope mismatch for $container_id." >&2
    exit 1
  fi
}

wait_for_web() {
  local container_id=$1
  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    local state health
    state=$(docker inspect --format '{{.State.Status}}' "$container_id")
    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \
      "$container_id")

    if [[ "$state" == "running" && "$health" == "healthy" ]]; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for web container $container_id to become healthy." >&2
  return 1
}

database_snapshot() {
  # Variables are intentionally expanded inside the isolated PostGIS container.
  # shellcheck disable=SC2016
  "${compose[@]}" exec -T db sh -c \
    'psql --no-psqlrc --tuples-only --no-align --set ON_ERROR_STOP=1 \
      --username "$POSTGRES_USER" --dbname "$POSTGRES_DB"' >"$1" <<'SQL'
BEGIN READ ONLY;
SELECT 'users' AS table_name, count(*) AS row_count FROM users
UNION ALL SELECT 'users_tokens', count(*) FROM users_tokens
UNION ALL SELECT 'help_requests', count(*) FROM help_requests
UNION ALL SELECT 'messages', count(*) FROM messages
UNION ALL SELECT 'categories', count(*) FROM categories
UNION ALL SELECT 'help_assignments', count(*) FROM help_assignments
UNION ALL SELECT 'activities', count(*) FROM activities
UNION ALL SELECT 'reports', count(*) FROM reports
UNION ALL SELECT 'social_identities', count(*) FROM social_identities
UNION ALL SELECT 'tracking_sessions', count(*) FROM tracking_sessions
UNION ALL SELECT 'tracking_positions', count(*) FROM tracking_positions
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations
ORDER BY table_name;
COMMIT;
SQL
}

published_url() {
  local service=$1
  local target_port=$2
  local published

  published=$("${compose[@]}" port "$service" "$target_port" | head -n 1)

  if [[ ! "$published" =~ ^127[.]0[.]0[.]1:([1-9][0-9]*)$ ]]; then
    echo "The $service port is not bound to an observed loopback port." >&2
    return 1
  fi

  printf 'http://%s' "$published"
}

wait_for_http() {
  local url=$1
  shift
  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    if curl --fail --silent --show-error "$@" "$url" >/dev/null 2>&1; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for $url." >&2
  return 1
}

fetch_receiver_events() {
  local receiver_id
  receiver_id=$(service_ids alert-receiver | head -n 1)
  assert_scope "$receiver_id" alert-receiver
  docker exec "$receiver_id" \
    python -c \
    'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/events", timeout=2).read().decode())'
}

wait_for_targets_up() {
  local expected_web=$1
  local expected_worker=$2
  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    if curl --fail --silent --show-error \
      "$prometheus_url/api/v1/targets?state=active" \
      >"$output_dir/targets-current.json" 2>/dev/null &&
      jq -e \
        --argjson expected_web "$expected_web" \
        --argjson expected_worker "$expected_worker" \
        --arg run_id "$LABEL" '
        [
          .data.activeTargets[]
          | select(
              .labels.job == "who-need-help-web" and
              .labels.run_id == $run_id
            )
        ] as $web_targets
        | [
            .data.activeTargets[]
            | select(
                .labels.job == "who-need-help-worker" and
                .labels.run_id == $run_id
              )
          ] as $worker_targets
        | ($web_targets | length) == $expected_web
          and ($worker_targets | length) == $expected_worker
          and all($web_targets[]; .health == "up")
          and all($worker_targets[]; .health == "up")
      ' "$output_dir/targets-current.json" >/dev/null; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for every direct web and worker metrics target." >&2
  return 1
}

wait_for_firing_alert() {
  local instance=$1
  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \
      >"$output_dir/prometheus-alerts-firing.json" 2>/dev/null &&
      jq -e --arg instance "$instance" --arg run_id "$LABEL" '
        any(
          .data.alerts[];
          .state == "firing" and
          .labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and
          .labels.instance == $instance and
          .labels.run_id == $run_id
        )
      ' "$output_dir/prometheus-alerts-firing.json" >/dev/null; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for the induced Prometheus alert." >&2
  return 1
}

wait_for_webhook_status() {
  local instance=$1
  local expected_status=$2
  local destination=$3
  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    if fetch_receiver_events >"$destination" 2>/dev/null &&
      jq -e \
        --arg instance "$instance" \
        --arg run_id "$LABEL" \
        --arg expected_status "$expected_status" '
          any(
            .[].payload.alerts[];
            .status == $expected_status and
            .labels.alertname == "WhoNeedHelpWebReplicaUnavailable" and
            .labels.instance == $instance and
            .labels.run_id == $run_id
          )
        ' "$destination" >/dev/null; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for the $expected_status Alertmanager webhook." >&2
  return 1
}

wait_for_alert_clear() {
  local instance=$1
  local deadline=$((SECONDS + OBSERVABILITY_TIMEOUT_SECONDS))

  while ((SECONDS < deadline)); do
    if curl --fail --silent --show-error "$prometheus_url/api/v1/alerts" \
      >"$output_dir/prometheus-alerts-resolved.json" 2>/dev/null &&
      jq -e --arg instance "$instance" --arg run_id "$LABEL" '
        all(
          .data.alerts[];
          .labels.alertname != "WhoNeedHelpWebReplicaUnavailable" or
          .labels.instance != $instance or
          .labels.run_id != $run_id or
          .state != "firing"
        )
      ' "$output_dir/prometheus-alerts-resolved.json" >/dev/null; then
      return 0
    fi

    sleep 1
  done

  echo "Timed out waiting for the induced Prometheus alert to resolve." >&2
  return 1
}

restore_web() {
  if [[ -n "${drill_web_id:-}" ]]; then
    local state
    state=$(docker inspect --format '{{.State.Status}}' "$drill_web_id" 2>/dev/null || true)

    if [[ "$state" != "running" ]]; then
      docker start "$drill_web_id" >/dev/null 2>&1 || true
    fi

    wait_for_web "$drill_web_id" >/dev/null 2>&1 || true
  fi
}

cleanup() {
  local status=$?
  trap - EXIT HUP INT TERM
  restore_web
  exit "$status"
}

trap cleanup EXIT HUP INT TERM

mapfile -t web_ids < <(service_ids web)
mapfile -t worker_ids < <(service_ids worker)

if [[ "${#web_ids[@]}" -ne "$LOAD_WEB_REPLICAS" ]]; then
  echo "Expected $LOAD_WEB_REPLICAS running load web replicas; observed ${#web_ids[@]}." >&2
  exit 1
fi

if [[ "${#worker_ids[@]}" -ne "$LOAD_WORKER_REPLICAS" ]]; then
  echo "Expected $LOAD_WORKER_REPLICAS running load worker replicas; observed ${#worker_ids[@]}." >&2
  exit 1
fi

internal_network=$(
  docker network ls \
    --filter "label=com.docker.compose.project=$LOAD_PROJECT" \
    --filter "label=com.docker.compose.network=internal" \
    --format '{{.Name}}'
)

if [[ -z "$internal_network" || "$internal_network" == *$'\n'* ]]; then
  echo "Expected exactly one observed Compose internal network." >&2
  exit 1
fi

target_lines="$prometheus_runtime/web-targets.jsonl"
: >"$target_lines"

for web_id in "${web_ids[@]}"; do
  assert_scope "$web_id" web
  wait_for_web "$web_id"
  web_name=$(docker inspect --format '{{.Name}}' "$web_id")
  web_name=${web_name#/}
  web_ip=$(
    docker inspect \
      --format "{{with index .NetworkSettings.Networks \"$internal_network\"}}{{.IPAddress}}{{end}}" \
      "$web_id"
  )

  if [[ ! "$web_ip" =~ ^[0-9]+([.][0-9]+){3}$ ]]; then
    echo "No observed IPv4 address for $web_name on $internal_network." >&2
    exit 1
  fi

  jq -cn \
    --arg target "$web_ip:4000" \
    --arg instance "$web_name" \
    --arg compose_project "$LOAD_PROJECT" \
    --arg run_id "$LABEL" \
    '{
      targets: [$target],
      labels: {
        instance: $instance,
        compose_project: $compose_project,
        run_id: $run_id
      }
    }' >>"$target_lines"
done

jq -s '.' "$target_lines" >"$prometheus_runtime/web-targets.json"
unlink "$target_lines"

target_lines="$prometheus_runtime/worker-targets.jsonl"
: >"$target_lines"

for worker_id in "${worker_ids[@]}"; do
  assert_scope "$worker_id" worker
  wait_for_web "$worker_id"
  worker_name=$(docker inspect --format '{{.Name}}' "$worker_id")
  worker_name=${worker_name#/}
  worker_ip=$(
    docker inspect \
      --format "{{with index .NetworkSettings.Networks \"$internal_network\"}}{{.IPAddress}}{{end}}" \
      "$worker_id"
  )

  if [[ ! "$worker_ip" =~ ^[0-9]+([.][0-9]+){3}$ ]]; then
    echo "No observed IPv4 address for $worker_name on $internal_network." >&2
    exit 1
  fi

  jq -cn \
    --arg target "$worker_ip:4000" \
    --arg instance "$worker_name" \
    --arg compose_project "$LOAD_PROJECT" \
    --arg run_id "$LABEL" \
    '{
      targets: [$target],
      labels: {
        instance: $instance,
        compose_project: $compose_project,
        run_id: $run_id
      }
    }' >>"$target_lines"
done

jq -s '.' "$target_lines" >"$prometheus_runtime/worker-targets.json"
unlink "$target_lines"

sed \
  -e "s/__SCRAPE_INTERVAL__/$OBSERVABILITY_SCRAPE_INTERVAL/g" \
  -e "s/__EVALUATION_INTERVAL__/$OBSERVABILITY_EVALUATION_INTERVAL/g" \
  "$ROOT/ops/observability/prometheus.template.yml" \
  >"$prometheus_runtime/prometheus.yml"
printf '%s' "$METRICS_TOKEN" >"$prometheus_runtime/metrics-token"
printf '%s' "$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
  >"$grafana_runtime/admin-password"
cp "$prometheus_runtime/web-targets.json" "$output_dir/generated-targets.json"
cp "$prometheus_runtime/worker-targets.json" "$output_dir/generated-worker-targets.json"
chmod 600 "$prometheus_runtime/prometheus.yml" \
  "$prometheus_runtime/web-targets.json" \
  "$prometheus_runtime/worker-targets.json" \
  "$prometheus_runtime/metrics-token" \
  "$grafana_runtime/admin-password" \
  "$output_dir/generated-targets.json" \
  "$output_dir/generated-worker-targets.json"
set_runtime_owner "$prometheus_runtime" "65534:65534"
set_runtime_owner "$grafana_runtime" "472:0"

docker run --rm \
  --volume "$prometheus_runtime:/runtime:ro" \
  --volume "$ROOT/ops/observability/rules.yml:/etc/prometheus/rules.yml:ro" \
  --entrypoint /bin/promtool \
  quay.io/prometheus/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 \
  check config /runtime/prometheus.yml \
  >"$output_dir/promtool-check.txt"

docker run --rm \
  --volume "$ROOT/ops/observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro" \
  --entrypoint /bin/amtool \
  quay.io/prometheus/alertmanager:v0.33.1@sha256:9e082985f56f4c8c9f724e18f2288c6708f472e56a5286b8863d080434ea065d \
  check-config /etc/alertmanager/alertmanager.yml \
  >"$output_dir/amtool-check.txt"

jq --exit-status 'type == "object" and .uid == "wnh-overview"' \
  "$ROOT/ops/observability/grafana/dashboards/who-need-help-overview.json" \
  >/dev/null

for service in alert-receiver alertmanager prometheus grafana; do
  while IFS= read -r existing_id; do
    [[ -n "$existing_id" ]] && assert_scope "$existing_id" "$service"
  done < <(service_ids "$service")
done

database_snapshot "$output_dir/database-before.txt"

"${compose[@]}" up -d --force-recreate --wait \
  alert-receiver alertmanager prometheus grafana \
  >"$output_dir/compose-up.txt"

for service in alert-receiver alertmanager prometheus grafana; do
  service_id=$(service_ids "$service" | head -n 1)
  assert_scope "$service_id" "$service"
done

prometheus_url=$(published_url prometheus 9090)
alertmanager_url=$(published_url alertmanager 9093)
grafana_url=$(published_url grafana 3000)

wait_for_http "$prometheus_url/-/ready"
wait_for_http "$alertmanager_url/-/ready"
wait_for_http "$grafana_url/api/health"
wait_for_http "$grafana_url/api/datasources/uid/wnh-prometheus/health" \
  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD"

curl --fail --silent --show-error \
  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
  "$grafana_url/api/health" >"$output_dir/grafana-health.json"
curl --fail --silent --show-error \
  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
  "$grafana_url/api/datasources/uid/wnh-prometheus/health" \
  >"$output_dir/grafana-datasource-health.json"
curl --fail --silent --show-error \
  --user "$OBSERVABILITY_GRAFANA_ADMIN_USER:$OBSERVABILITY_GRAFANA_ADMIN_PASSWORD" \
  "$grafana_url/api/dashboards/uid/wnh-overview" \
  >"$output_dir/grafana-dashboard.json"

if ! jq -e '
  .dashboard.uid == "wnh-overview" and
  .meta.provisioned == true and
  (.dashboard.panels | length) == 10
' "$output_dir/grafana-dashboard.json" >/dev/null; then
  echo "The provisioned Grafana dashboard did not match the tracked dashboard." >&2
  exit 1
fi

wait_for_targets_up "$LOAD_WEB_REPLICAS" "$LOAD_WORKER_REPLICAS"
mv "$output_dir/targets-current.json" "$output_dir/targets-before-drill.json"

jq -n \
  --slurpfile expected_web "$output_dir/generated-targets.json" \
  --slurpfile expected_worker "$output_dir/generated-worker-targets.json" \
  --slurpfile observed "$output_dir/targets-before-drill.json" '
    {
      expected_web_instances: ($expected_web[0] | map(.labels.instance) | sort),
      observed_web_instances: (
        $observed[0].data.activeTargets
        | map(select(.labels.job == "who-need-help-web") | .labels.instance)
        | sort
      ),
      expected_worker_instances: ($expected_worker[0] | map(.labels.instance) | sort),
      observed_worker_instances: (
        $observed[0].data.activeTargets
        | map(select(.labels.job == "who-need-help-worker") | .labels.instance)
        | sort
      )
    }
    | . + {
        exact_instance_match:
          (.expected_web_instances == .observed_web_instances and
           .expected_worker_instances == .observed_worker_instances)
      }
  ' >"$output_dir/target-summary.json"

if ! jq -e '.exact_instance_match' "$output_dir/target-summary.json" >/dev/null; then
  echo "Prometheus did not preserve every direct web and worker instance target." >&2
  exit 1
fi

drill_web_id=${web_ids[0]}
assert_scope "$drill_web_id" web
drill_instance=$(docker inspect --format '{{.Name}}' "$drill_web_id")
drill_instance=${drill_instance#/}
docker stop --time 30 "$drill_web_id" >"$output_dir/stopped-web.txt"

wait_for_firing_alert "$drill_instance"
wait_for_webhook_status \
  "$drill_instance" firing "$output_dir/alert-webhooks-firing.json"

docker start "$drill_web_id" >"$output_dir/started-web.txt"
wait_for_web "$drill_web_id"
wait_for_targets_up "$LOAD_WEB_REPLICAS" "$LOAD_WORKER_REPLICAS"
mv "$output_dir/targets-current.json" "$output_dir/targets-after-recovery.json"
wait_for_alert_clear "$drill_instance"
wait_for_webhook_status \
  "$drill_instance" resolved "$output_dir/alert-webhooks-complete.json"

curl --fail --silent --show-error "$alertmanager_url/api/v2/alerts" \
  >"$output_dir/alertmanager-alerts-after-recovery.json"
database_snapshot "$output_dir/database-after.txt"

if ! diff -u "$output_dir/database-before.txt" "$output_dir/database-after.txt" \
  >"$output_dir/database-diff.txt"; then
  echo "The observability drill changed tracked database counts." >&2
  exit 1
fi

"${compose[@]}" ps -a >"$output_dir/compose-after.txt"
"${compose[@]}" images --format json >"$output_dir/images.json"

jq -n \
  --arg run_id "$LABEL" \
  --arg drill_instance "$drill_instance" \
  --arg prometheus_url "$prometheus_url" \
  --arg alertmanager_url "$alertmanager_url" \
  --arg grafana_url "$grafana_url" \
  --argjson web_replicas "$LOAD_WEB_REPLICAS" \
  --argjson worker_replicas "$LOAD_WORKER_REPLICAS" \
  '{
    run_id: $run_id,
    web_replicas: $web_replicas,
    worker_replicas: $worker_replicas,
    direct_targets_up_before_and_after: true,
    induced_instance: $drill_instance,
    firing_webhook_observed: true,
    resolved_webhook_observed: true,
    database_count_diff_bytes: 0,
    prometheus_url: $prometheus_url,
    alertmanager_url: $alertmanager_url,
    grafana_url: $grafana_url,
    grafana_dashboard_path: "/d/wnh-overview/overview"
  }' >"$output_dir/summary.json"

trap - EXIT HUP INT TERM

printf 'Observability evidence: %s\n' "$output_dir"
printf 'Prometheus: %s\nAlertmanager: %s\nGrafana: %s/d/wnh-overview/overview\n' \
  "$prometheus_url" "$alertmanager_url" "$grafana_url"
printf 'Grafana user: %s; its random password remains only in ignored .env.load.\n' \
  "$OBSERVABILITY_GRAFANA_ADMIN_USER"